diff --git a/.devcontainer/wasi/devcontainer.json b/.devcontainer/wasi/devcontainer.json new file mode 100644 index 00000000000..4266144ce47 --- /dev/null +++ b/.devcontainer/wasi/devcontainer.json @@ -0,0 +1,73 @@ +{ + "image": "ghcr.io/python/wasicontainer:latest", + "onCreateCommand": [ + // Install common tooling. + "dnf", + "install", + "-y", + // For umask fix below. + "/usr/bin/setfacl" + ], + "updateContentCommand": { + // Using the shell for `nproc` usage. + "python": "python3 Tools/wasm/wasi build --quiet -- --with-pydebug -C" + }, + "postCreateCommand": { + // https://github.com/orgs/community/discussions/26026 + "umask fix: workspace": ["sudo", "setfacl", "-bnR", "."], + "umask fix: /tmp": ["sudo", "setfacl", "-bnR", "/tmp"] + }, + "customizations": { + "vscode": { + "extensions": [ + // Highlighting for Parser/Python.asdl. + "brettcannon.zephyr-asdl", + // Highlighting for configure.ac. + "maelvalais.autoconf", + // C auto-complete. + "ms-vscode.cpptools", + // Python auto-complete. + "ms-python.python" + ], + "settings": { + "C_Cpp.default.compilerPath": "/usr/bin/clang", + "C_Cpp.default.cStandard": "c11", + "C_Cpp.default.defines": [ + "CONFIG_64", + "Py_BUILD_CORE" + ], + "C_Cpp.default.includePath": [ + "${workspaceFolder}/*", + "${workspaceFolder}/Include/**" + ], + // https://github.com/microsoft/vscode-cpptools/issues/10732 + "C_Cpp.errorSquiggles": "disabled", + "editor.insertSpaces": true, + "editor.rulers": [ + 80 + ], + "editor.tabSize": 4, + "editor.trimAutoWhitespace": true, + "files.associations": { + "*.h": "c" + }, + "files.encoding": "utf8", + "files.eol": "\n", + "files.insertFinalNewline": true, + "files.trimTrailingWhitespace": true, + "python.analysis.diagnosticSeverityOverrides": { + // Complains about shadowing the stdlib w/ the stdlib. + "reportShadowedImports": "none", + // Doesn't like _frozen_importlib. + "reportMissingImports": "none" + }, + "python.analysis.extraPaths": [ + "Lib" + ], + "[restructuredtext]": { + "editor.tabSize": 3 + } + } + } + } +} diff --git a/.gitattributes b/.gitattributes index 823e3e975a2..f4d65dfd1df 100644 --- a/.gitattributes +++ b/.gitattributes @@ -34,6 +34,9 @@ Lib/test/xmltestdata/* noeol Lib/venv/scripts/common/activate text eol=lf Lib/venv/scripts/posix/* text eol=lf +# Prevent GitHub's web conflict editor from converting LF to CRLF +*.rst text eol=lf + # CRLF files [attr]dos text eol=crlf @@ -68,6 +71,7 @@ PCbuild/readme.txt dos **/clinic/*.cpp.h generated **/clinic/*.h.h generated *_db.h generated +Doc/_static/tachyon-example-*.html generated Doc/c-api/lifecycle.dot.svg generated Doc/data/stable_abi.dat generated Doc/library/token-list.inc generated @@ -82,13 +86,19 @@ Include/opcode.h generated Include/opcode_ids.h generated Include/token.h generated Lib/_opcode_metadata.py generated +Lib/idlelib/help.html generated Lib/keyword.py generated +Lib/pydoc_data/topics.py generated +Lib/pydoc_data/module_docs.py generated Lib/test/certdata/*.pem generated Lib/test/certdata/*.0 generated Lib/test/levenshtein_examples.json generated Lib/test/test_stable_abi_ctypes.py generated +Lib/test/test_zoneinfo/data/*.json generated Lib/token.py generated Misc/sbom.spdx.json generated +Modules/_testinternalcapi/test_cases.c.h generated +Modules/_testinternalcapi/test_targets.h generated Objects/typeslots.inc generated PC/python3dll.c generated Parser/parser.c generated @@ -99,8 +109,10 @@ Python/executor_cases.c.h generated Python/generated_cases.c.h generated Python/optimizer_cases.c.h generated Python/opcode_targets.h generated +Python/record_functions.c.h generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated aclocal.m4 generated configure generated *.min.js generated +package-lock.json generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0928f051d51..af904a567cf 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -63,9 +63,10 @@ .azure-pipelines/ @AA-Turner # GitHub & related scripts -.github/ @ezio-melotti @hugovk @AA-Turner -Tools/build/compute-changes.py @AA-Turner -Tools/build/verify_ensurepip_wheels.py @AA-Turner @pfmoore @pradyunsg +.github/ @ezio-melotti @hugovk @AA-Turner @webknjaz +Tools/build/compute-changes.py @AA-Turner @hugovk @webknjaz +Lib/test/test_tools/test_compute_changes.py @AA-Turner @hugovk @webknjaz +Tools/build/verify_ensurepip_wheels.py @AA-Turner @pfmoore @pradyunsg # Pre-commit .pre-commit-config.yaml @hugovk @@ -82,10 +83,14 @@ Tools/patchcheck/ @AA-Turner # Autotools configure* @erlend-aasland @corona10 @AA-Turner @emmatyping Makefile.pre.in @erlend-aasland @AA-Turner @emmatyping +Modules/makesetup @erlend-aasland @AA-Turner @emmatyping Modules/Setup* @erlend-aasland @AA-Turner @emmatyping -Modules/makesetup @emmatyping Tools/build/regen-configure.sh @AA-Turner +# generate-build-details +Tools/build/generate-build-details.py @FFY00 +Lib/test/test_build_details.py @FFY00 + # ---------------------------------------------------------------------------- # Documentation @@ -95,17 +100,18 @@ Tools/build/regen-configure.sh @AA-Turner InternalDocs/ @AA-Turner # Tools, Configuration, etc -Doc/Makefile @AA-Turner @hugovk -Doc/_static/ @AA-Turner @hugovk -Doc/conf.py @AA-Turner @hugovk -Doc/make.bat @AA-Turner @hugovk -Doc/requirements.txt @AA-Turner @hugovk -Doc/tools/ @AA-Turner @hugovk +Doc/Makefile @AA-Turner @hugovk @StanFromIreland +Doc/_static/ @AA-Turner @hugovk @StanFromIreland +Doc/conf.py @AA-Turner @hugovk @StanFromIreland +Doc/make.bat @AA-Turner @hugovk @StanFromIreland +Doc/requirements.txt @AA-Turner @hugovk @StanFromIreland +Doc/tools/ @AA-Turner @hugovk @StanFromIreland # PR Previews .readthedocs.yml @AA-Turner # Sections +Doc/c-api/ @ZeroIntensity Doc/reference/ @willingc @AA-Turner Doc/whatsnew/ @AA-Turner @@ -122,8 +128,13 @@ Doc/howto/clinic.rst @erlend-aasland @AA-Turner # C Analyser Tools/c-analyzer/ @ericsnowcurrently +# C API Documentation Checks +Tools/check-c-api-docs/ @ZeroIntensity + # Fuzzing -Modules/_xxtestfuzz/ @ammaraskar +Modules/_xxtestfuzz/ @python/fuzzers +Lib/test/test_xxtestfuzz.py @python/fuzzers +.github/workflows/reusable-cifuzz.yml @python/fuzzers # Limited C API & Stable ABI Doc/c-api/stable.rst @encukou @@ -136,6 +147,9 @@ Misc/externals.spdx.json @sethmlarson Misc/sbom.spdx.json @sethmlarson Tools/build/generate_sbom.py @sethmlarson +# ABI check +Misc/libabigail.abignore @encukou + # ---------------------------------------------------------------------------- # Platform Support @@ -150,6 +164,7 @@ Lib/test/test_android.py @mhsmith @freakboy3742 # iOS Doc/using/ios.rst @freakboy3742 Lib/_ios_support.py @freakboy3742 +Apple/ @freakboy3742 iOS/ @freakboy3742 # macOS @@ -165,9 +180,10 @@ Tools/wasm/config.site-wasm32-emscripten @freakboy3742 @emmatyping Tools/wasm/emscripten @freakboy3742 @emmatyping # WebAssembly (WASI) -Tools/wasm/wasi-env @brettcannon @emmatyping -Tools/wasm/wasi.py @brettcannon @emmatyping -Tools/wasm/wasi @brettcannon @emmatyping +Platforms/WASI @brettcannon @emmatyping @savannahostrowski +Tools/wasm/wasi-env @brettcannon @emmatyping @savannahostrowski +Tools/wasm/wasi.py @brettcannon @emmatyping @savannahostrowski +Tools/wasm/wasi @brettcannon @emmatyping @savannahostrowski # Windows PC/ @python/windows-team @@ -240,51 +256,57 @@ Lib/test/test_getpath.py @FFY00 Modules/getpath* @FFY00 # Hashing / ``hash()`` and related -Include/cpython/pyhash.h @gpshead @picnixz @tiran -Include/internal/pycore_pyhash.h @gpshead @picnixz @tiran -Include/pyhash.h @gpshead @picnixz @tiran -Python/pyhash.c @gpshead @picnixz @tiran +Include/cpython/pyhash.h @gpshead @picnixz +Include/internal/pycore_pyhash.h @gpshead @picnixz +Include/pyhash.h @gpshead @picnixz +Python/pyhash.c @gpshead @picnixz # The import system (including importlib) -**/*import* @brettcannon @ericsnowcurrently @ncoghlan @warsaw -Python/import.c @brettcannon @ericsnowcurrently @ncoghlan @warsaw @kumaraditya303 +**/*import* @brettcannon @ericsnowcurrently @ncoghlan @warsaw @FFY00 +Python/import.c @brettcannon @ericsnowcurrently @ncoghlan @warsaw @FFY00 @kumaraditya303 **/*freeze* @ericsnowcurrently **/*frozen* @ericsnowcurrently **/*modsupport* @ericsnowcurrently -**/*modulefinder* @ericsnowcurrently +**/*modulefinder* @ericsnowcurrently @FFY00 **/*moduleobject* @ericsnowcurrently **/*multiphase* @ericsnowcurrently -**/*pkgutil* @ericsnowcurrently +**/*pkgutil* @ericsnowcurrently @FFY00 **/*pythonrun* @ericsnowcurrently -**/*runpy* @ericsnowcurrently +**/*runpy* @ericsnowcurrently @FFY00 **/*singlephase* @ericsnowcurrently Doc/c-api/module.rst @ericsnowcurrently Lib/test/test_module/ @ericsnowcurrently -Python/dynload_*.c @ericsnowcurrently +Python/dynload_*.c @ericsnowcurrently @FFY00 # Initialisation -**/*initconfig* @ericsnowcurrently -**/*pathconfig* @ericsnowcurrently -**/*preconfig* @ericsnowcurrently +**/*initconfig* @ericsnowcurrently @FFY00 +**/*pathconfig* @ericsnowcurrently @FFY00 +**/*preconfig* @ericsnowcurrently @FFY00 Doc/library/sys_path_init.rst @FFY00 Doc/c-api/init_config.rst @FFY00 # Interpreter main program -Modules/main.c @ericsnowcurrently -Programs/_bootstrap_python.c @ericsnowcurrently -Programs/python.c @ericsnowcurrently +Modules/main.c @ericsnowcurrently @FFY00 +Programs/_bootstrap_python.c @ericsnowcurrently @FFY00 +Programs/python.c @ericsnowcurrently @FFY00 # JIT +.github/workflows/jit.yml @savannahostrowski Include/internal/pycore_jit.h @brandtbucher @savannahostrowski @diegorusso Python/jit.c @brandtbucher @savannahostrowski @diegorusso Tools/jit/ @brandtbucher @savannahostrowski @diegorusso InternalDocs/jit.md @brandtbucher @savannahostrowski @diegorusso @AA-Turner +# Lazy imports (PEP 810) +Objects/lazyimportobject.c @yhg1s @DinoV @pablogsal +Include/internal/pycore_lazyimportobject.h @yhg1s @DinoV @pablogsal +Lib/test/test_lazy_import @yhg1s @DinoV @pablogsal + # Micro-op / μop / Tier 2 Optimiser -Python/optimizer.c @markshannon -Python/optimizer_analysis.c @markshannon @tomasr8 @Fidget-Spinner -Python/optimizer_bytecodes.c @markshannon @tomasr8 @Fidget-Spinner -Python/optimizer_symbols.c @markshannon @tomasr8 +Python/optimizer.c @markshannon @Fidget-Spinner +Python/optimizer_analysis.c @markshannon @tomasr8 @Fidget-Spinner @savannahostrowski +Python/optimizer_bytecodes.c @markshannon @tomasr8 @Fidget-Spinner @savannahostrowski +Python/optimizer_symbols.c @markshannon @tomasr8 @Fidget-Spinner @savannahostrowski # Parser, Lexer, and Grammar Grammar/python.gram @pablogsal @lysnikolaou @@ -296,8 +318,8 @@ Tools/peg_generator/ @pablogsal @lysnikolaou # Runtime state/lifecycle **/*gil* @ericsnowcurrently -**/*pylifecycle* @ericsnowcurrently @ZeroIntensity -**/*pystate* @ericsnowcurrently @ZeroIntensity +**/*pylifecycle* @ericsnowcurrently @ZeroIntensity @FFY00 +**/*pystate* @ericsnowcurrently @ZeroIntensity @FFY00 Include/internal/pycore_*_init.h @ericsnowcurrently Include/internal/pycore_*_state.h @ericsnowcurrently Include/internal/pycore_atexit.h @ericsnowcurrently @@ -314,7 +336,7 @@ Tools/build/generate_global_objects.py @ericsnowcurrently # Remote Debugging Python/remote_debug.h @pablogsal Python/remote_debugging.c @pablogsal -Modules/_remote_debugging_module.c @pablogsal @ambv @1st1 +Modules/_remote_debugging/ @pablogsal # Sub-Interpreters **/*crossinterp* @ericsnowcurrently @@ -370,14 +392,14 @@ Lib/calendar.py @AA-Turner Lib/test/test_calendar.py @AA-Turner # Cryptographic Primitives and Applications -**/*hashlib* @gpshead @picnixz @tiran -**/*hashopenssl* @gpshead @picnixz @tiran +**/*hashlib* @gpshead @picnixz +**/*hashopenssl* @gpshead @picnixz **/*hmac* @gpshead @picnixz **/*ssl* @gpshead @picnixz Modules/_hacl/ @gpshead @picnixz -Modules/*blake* @gpshead @picnixz @tiran -Modules/*md5* @gpshead @picnixz @tiran -Modules/*sha* @gpshead @picnixz @tiran +Modules/*blake* @gpshead @picnixz +Modules/*md5* @gpshead @picnixz +Modules/*sha* @gpshead @picnixz # Codecs Modules/cjkcodecs/ @corona10 @@ -405,14 +427,19 @@ Lib/dataclasses.py @ericvsmith Lib/test/test_dataclasses/ @ericvsmith # Dates and times -Doc/**/*time.rst @pganssle @abalkin -Include/datetime.h @pganssle @abalkin -Include/internal/pycore_time.h @pganssle @abalkin -Lib/*time.py @pganssle @abalkin -Lib/test/datetimetester.py @pganssle @abalkin -Lib/test/test_*time.py @pganssle @abalkin -Modules/*time* @pganssle @abalkin -Python/pytime.c @pganssle @abalkin +Doc/**/*time.rst @pganssle @StanFromIreland +Doc/library/datetime-* @pganssle @StanFromIreland +Doc/library/zoneinfo.rst @pganssle @StanFromIreland +Include/datetime.h @pganssle @StanFromIreland +Include/internal/pycore_time.h @pganssle @StanFromIreland +Lib/test/test_zoneinfo/ @pganssle @StanFromIreland +Lib/zoneinfo/ @pganssle @StanFromIreland +Lib/*time.py @pganssle @StanFromIreland +Lib/test/datetimetester.py @pganssle @StanFromIreland +Lib/test/test_*time.py @pganssle @StanFromIreland +Modules/*zoneinfo* @pganssle @StanFromIreland +Modules/*time* @pganssle @StanFromIreland +Python/pytime.c @pganssle @StanFromIreland # Dbm Doc/library/dbm.rst @corona10 @erlend-aasland @serhiy-storchaka @@ -451,8 +478,9 @@ Lib/test/test_functools.py @rhettinger Modules/_functoolsmodule.c @rhettinger # Garbage collector -Modules/gcmodule.c @pablogsal -Doc/library/gc.rst @pablogsal +Modules/gcmodule.c @pablogsal +Doc/library/gc.rst @pablogsal +InternalDocs/garbage_collector.md @pablogsal # Gettext Doc/library/gettext.rst @tomasr8 @@ -479,13 +507,13 @@ Lib/idlelib/ @terryjreedy Lib/turtledemo/ @terryjreedy # importlib.metadata -Doc/library/importlib.metadata.rst @jaraco @warsaw -Lib/importlib/metadata/ @jaraco @warsaw -Lib/test/test_importlib/metadata/ @jaraco @warsaw +Doc/library/importlib.metadata.rst @jaraco @warsaw @FFY00 +Lib/importlib/metadata/ @jaraco @warsaw @FFY00 +Lib/test/test_importlib/metadata/ @jaraco @warsaw @FFY00 # importlib.resources -Doc/library/importlib.resources.abc.rst @jaraco @warsaw -Doc/library/importlib.resources.rst @jaraco @warsaw +Doc/library/importlib.resources.abc.rst @jaraco @warsaw @FFY00 +Doc/library/importlib.resources.rst @jaraco @warsaw @FFY00 Lib/importlib/resources/ @jaraco @warsaw @FFY00 Lib/test/test_importlib/resources/ @jaraco @warsaw @FFY00 @@ -525,6 +553,11 @@ Lib/pydoc.py @AA-Turner Lib/pydoc_data/ @AA-Turner Lib/test/test_pydoc/ @AA-Turner +# Profiling (Sampling) +Doc/library/profiling*.rst @pablogsal +Lib/profiling/ @pablogsal +Lib/test/test_profiling/ @pablogsal + # PyREPL Lib/_pyrepl/ @pablogsal @lysnikolaou @ambv Lib/test/test_pyrepl/ @pablogsal @lysnikolaou @ambv diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index 5b86302bdd1..94b67ce3dbe 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -28,23 +28,23 @@ Please be aware that our workflow does deviate slightly from the typical GitHub project. Details on how to properly submit a pull request are covered in `Lifecycle of a Pull Request `_. We utilize various bots and status checks to help with this, so do follow the -comments they leave and their "Details" links, respectively. The key points of -our workflow that are not covered by a bot or status check are: +comments they leave and their "Details" links, respectively. -- All discussions that are not directly related to the code in the pull request - should happen on `GitHub Issues `_. -- Upon your first non-trivial pull request (which includes documentation changes), - feel free to add yourself to ``Misc/ACKS`` +The final key part of our workflow is that all discussions that are not +directly related to the code in the pull request should happen on +`GitHub Issues `__, generally in the +pull request's parent issue. Setting Expectations -------------------- -Due to the fact that this project is entirely volunteer-run (i.e. no one is paid -to work on Python full-time), we unfortunately can make no guarantees as to if +Due to the fact that this project is run by volunteers, +unfortunately we cannot make any guarantees as to if or when a core developer will get around to reviewing your pull request. If no core developer has done a review or responded to changes made because of a -"changes requested" review, please feel free to email python-dev to ask if -someone could take a look at your pull request. +"changes requested" review within a month, you can ask for someone to +review your pull request via a post in the `Core Development Discourse +category `__. Code of Conduct diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 75d174307ce..de6e8756b03 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -5,3 +5,6 @@ contact_links: - name: "Proposing new features" about: "Submit major feature proposal (e.g. syntax changes) to an ideas forum first." url: "https://discuss.python.org/c/ideas/6" + - name: "Python Install Manager issues" + about: "Report issues with the Python Install Manager (for Windows)" + url: "https://github.com/python/pymanager/issues" diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md deleted file mode 100644 index 174fd39171d..00000000000 --- a/.github/ISSUE_TEMPLATE/documentation.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -name: Documentation -about: Report a problem with the documentation -labels: "docs" ---- - -# Documentation - -(A clear and concise description of the issue.) diff --git a/.github/ISSUE_TEMPLATE/documentation.yml b/.github/ISSUE_TEMPLATE/documentation.yml new file mode 100644 index 00000000000..d720cf9c4de --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.yml @@ -0,0 +1,16 @@ +name: Documentation +description: Report a problem with the documentation +labels: ["docs"] +body: + - type: markdown + attributes: + value: | + > [!NOTE] + > Trivial changes (for example typos) don’t require an issue before opening a PR. + - type: textarea + id: description + attributes: + label: "Documentation" + description: "A clear and concise description of the issue. Include a link to the page." + validations: + required: true diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 68aae196357..eacfff24889 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,7 +1,3 @@ -self-hosted-runner: - # Pending https://github.com/rhysd/actionlint/issues/533 - labels: ["windows-11-arm"] - config-variables: null paths: diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c8a3165d690..4b77646e22d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,7 +3,7 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "monthly" + interval: "quarterly" labels: - "skip issue" - "skip news" @@ -12,10 +12,21 @@ updates: update-types: - "version-update:semver-minor" - "version-update:semver-patch" + groups: + actions: + patterns: + - "*" + cooldown: + # https://blog.yossarian.net/2025/11/21/We-should-all-be-using-dependency-cooldowns + # Cooldowns protect against supply chain attacks by avoiding the + # highest-risk window immediately after new releases. + default-days: 14 - package-ecosystem: "pip" directory: "/Tools/" schedule: - interval: "monthly" + interval: "quarterly" labels: - "skip issue" - "skip news" + cooldown: + default-days: 14 diff --git a/.github/workflows/add-issue-header.yml b/.github/workflows/add-issue-header.yml index 3cbc23af578..4c25976b9c2 100644 --- a/.github/workflows/add-issue-header.yml +++ b/.github/workflows/add-issue-header.yml @@ -12,6 +12,8 @@ on: # Only ever run once - opened +permissions: + contents: read jobs: add-header: @@ -20,7 +22,7 @@ jobs: issues: write timeout-minutes: 5 steps: - - uses: actions/github-script@v7 + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: # language=JavaScript script: | diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 83a668fc720..33a5950c148 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,7 +64,7 @@ jobs: run: | apt update && apt install git -yq git config --global --add safe.directory "$GITHUB_WORKSPACE" - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 1 persist-credentials: false @@ -101,28 +101,16 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.x' - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}-${{ env.pythonLocation }} - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Add ccache to PATH - run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Configure CPython run: | # Build Python with the libpython dynamic library @@ -153,6 +141,14 @@ jobs: if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME run: make check-c-globals + check-c-api-docs: + name: C API Docs + needs: build-context + if: >- + needs.build-context.outputs.run-tests == 'true' + || needs.build-context.outputs.run-docs == 'true' + uses: ./.github/workflows/reusable-check-c-api-docs.yml + build-windows: name: >- Windows @@ -169,13 +165,21 @@ jobs: free-threading: - false - true + interpreter: + - switch-case exclude: # Skip Win32 on free-threaded builds - { arch: Win32, free-threading: true } + include: + # msvc::musttail is currently only supported on x64, + # and only supported on 3.15+. + - { arch: x64, free-threading: false, interpreter: tail-call } + - { arch: x64, free-threading: true, interpreter: tail-call } uses: ./.github/workflows/reusable-windows.yml with: arch: ${{ matrix.arch }} free-threading: ${{ matrix.free-threading }} + interpreter: ${{ matrix.interpreter }} build-windows-msi: # ${{ '' } is a hack to nest jobs under the same sidebar category. @@ -198,32 +202,23 @@ jobs: macOS ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-macos == 'true' strategy: fail-fast: false matrix: - # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. - # macOS 13 only runs tests against the GIL-enabled CPython. - # Cirrus used for upstream, macos-14 for forks. + # macos-26 is Apple Silicon, macos-15-intel is Intel. + # macos-15-intel only runs tests against the GIL-enabled CPython. os: - - ghcr.io/cirruslabs/macos-runner:sonoma - - macos-14 - - macos-13 - is-fork: # only used for the exclusion trick - - ${{ github.repository_owner != 'python' }} + - macos-26 + - macos-15-intel free-threading: - false - true exclude: - - os: ghcr.io/cirruslabs/macos-runner:sonoma - is-fork: true - - os: macos-14 - is-fork: false - - os: macos-13 + - os: macos-15-intel free-threading: true uses: ./.github/workflows/reusable-macos.yml with: - config_hash: ${{ needs.build-context.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} @@ -233,7 +228,7 @@ jobs: ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} ${{ fromJSON(matrix.bolt) && '(bolt)' || '' }} needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' strategy: fail-fast: false matrix: @@ -253,195 +248,160 @@ jobs: # BOLT currently crashes during instrumentation on aarch64 - os: ubuntu-24.04-arm bolt: true + include: + # Enable CPU-intensive tests on ARM (default build only) + - os: ubuntu-24.04-arm + bolt: false + free-threading: false + test-opts: '-u cpu' uses: ./.github/workflows/reusable-ubuntu.yml with: - config_hash: ${{ needs.build-context.outputs.config-hash }} bolt-optimizations: ${{ matrix.bolt }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} + test-opts: ${{ matrix.test-opts || '' }} - build-ubuntu-ssltests-openssl: - name: 'Ubuntu SSL tests with OpenSSL' + build-ubuntu-ssltests: + name: 'Ubuntu SSL tests' runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' strategy: fail-fast: false matrix: os: [ubuntu-24.04] - # Keep 1.1.1w in our list despite it being upstream EOL and otherwise - # unsupported as it most resembles other 1.1.1-work-a-like ssl APIs - # supported by important vendors such as AWS-LC. - openssl_ver: [1.1.1w, 3.0.17, 3.2.5, 3.3.4, 3.4.2, 3.5.2] - # See Tools/ssl/make_ssl_data.py for notes on adding a new version + ssllib: + # See Tools/ssl/make_ssl_data.py for notes on adding a new version + ## OpenSSL + # Keep 1.1.1w in our list despite it being upstream EOL and otherwise + # unsupported as it most resembles other 1.1.1-work-a-like ssl APIs + # supported by important vendors such as AWS-LC. + - { name: openssl, version: 1.1.1w } + - { name: openssl, version: 3.0.19 } + - { name: openssl, version: 3.3.6 } + - { name: openssl, version: 3.4.4 } + - { name: openssl, version: 3.5.5 } + - { name: openssl, version: 3.6.1 } + ## AWS-LC + - { name: aws-lc, version: 1.68.0 } env: - OPENSSL_VER: ${{ matrix.openssl_ver }} + SSLLIB_VER: ${{ matrix.ssllib.version }} MULTISSL_DIR: ${{ github.workspace }}/multissl - OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }} - LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib + SSLLIB_DIR: ${{ github.workspace }}/multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }} + LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }}/lib steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v4 + - name: 'Restore SSL library build' + id: cache-ssl-lib + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - - name: Configure CPython - run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$OPENSSL_DIR" - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: SSL tests - run: ./python Lib/test/ssltests.py - - build-ubuntu-ssltests-awslc: - name: 'Ubuntu SSL tests with AWS-LC' - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04] - awslc_ver: [1.55.0] - env: - AWSLC_VER: ${{ matrix.awslc_ver}} - MULTISSL_DIR: ${{ github.workspace }}/multissl - OPENSSL_DIR: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }} - LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}/lib - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure SSL lib env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore AWS-LC build' - id: cache-aws-lc - uses: actions/cache@v4 - with: - path: ./multissl/aws-lc/${{ matrix.awslc_ver }} - key: ${{ matrix.os }}-multissl-aws-lc-${{ matrix.awslc_ver }} - - name: Install AWS-LC - if: steps.cache-aws-lc.outputs.cache-hit != 'true' + path: ./multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }} + key: ${{ matrix.os }}-multissl-${{ matrix.ssllib.name }}-${{ matrix.ssllib.version }} + - name: Install SSL Library + if: steps.cache-ssl-lib.outputs.cache-hit != 'true' run: | python3 Tools/ssl/multissltests.py \ --steps=library \ --base-directory "$MULTISSL_DIR" \ - --awslc ${{ matrix.awslc_ver }} \ + '--${{ matrix.ssllib.name }}' '${{ matrix.ssllib.version }}' \ --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Configure CPython run: | ./configure CFLAGS="-fdiagnostics-format=json" \ --config-cache \ --enable-slower-safety \ --with-pydebug \ - --with-openssl="$OPENSSL_DIR" \ + --with-openssl="$SSLLIB_DIR" \ --with-builtin-hashlib-hashes=blake2 \ --with-ssl-default-suites=openssl - name: Build CPython - run: make -j + run: make -j4 - name: Display build info run: make pythoninfo - - name: Verify python is linked to AWS-LC - run: ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC + - name: Verify python is linked to the right lib + run: | + ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' \ + | grep -iE '${{ matrix.ssllib.name }}.*${{ matrix.ssllib.version }}' - name: SSL tests run: ./python Lib/test/ssltests.py build-android: name: Android (${{ matrix.arch }}) needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-android == 'true' timeout-minutes: 60 strategy: fail-fast: false matrix: include: - # Use the same runs-on configuration as build-macos and build-ubuntu. - arch: aarch64 - runs-on: ${{ github.repository_owner == 'python' && 'ghcr.io/cirruslabs/macos-runner:sonoma' || 'macos-14' }} + runs-on: macos-26 - arch: x86_64 runs-on: ubuntu-24.04 runs-on: ${{ matrix.runs-on }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Build and test - run: ./Android/android.py ci ${{ matrix.arch }}-linux-android + run: python3 Platforms/Android ci --fast-ci ${{ matrix.arch }}-linux-android + + build-ios: + name: iOS + needs: build-context + if: needs.build-context.outputs.run-ios == 'true' + timeout-minutes: 60 + runs-on: macos-14 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + # GitHub recommends explicitly selecting the desired Xcode version: + # https://github.com/actions/runner-images/issues/12541#issuecomment-3083850140 + # This became a necessity as a result of + # https://github.com/actions/runner-images/issues/12541 and + # https://github.com/actions/runner-images/issues/12751. + - name: Select Xcode version + run: | + sudo xcode-select --switch /Applications/Xcode_15.4.app + + - name: Build and test + run: python3 Platforms/Apple ci iOS --fast-ci --simulator 'iPhone SE (3rd generation),OS=17.5' + + build-emscripten: + name: 'Emscripten' + needs: build-context + if: needs.build-context.outputs.run-emscripten == 'true' + uses: ./.github/workflows/reusable-emscripten.yml build-wasi: name: 'WASI' needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-wasi == 'true' uses: ./.github/workflows/reusable-wasi.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} test-hypothesis: name: "Hypothesis tests on Ubuntu" runs-on: ubuntu-24.04 timeout-minutes: 60 needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' env: - OPENSSL_VER: 3.0.16 + OPENSSL_VER: 3.5.5 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Register gcc problem matcher @@ -455,20 +415,13 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" @@ -479,11 +432,6 @@ jobs: run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -514,7 +462,7 @@ jobs: ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" - name: 'Restore Hypothesis database' id: cache-hypothesis-database - uses: actions/cache@v4 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ key: hypothesis-database-${{ github.head_ref || github.run_id }} @@ -541,7 +489,7 @@ jobs: -x test_subprocess \ -x test_signal \ -x test_sysconfig - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 if: always() with: name: hypothesis-example-db @@ -552,32 +500,27 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' strategy: fail-fast: false matrix: os: [ubuntu-24.04] env: - OPENSSL_VER: 3.0.16 + OPENSSL_VER: 3.5.5 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Set up GCC-10 for ASAN - uses: egor-tensin/setup-gcc@v1 + uses: egor-tensin/setup-gcc@a2861a8b8538f49cf2850980acccf6b05a1b2ae4 # v2.0 with: version: 10 - name: Configure OpenSSL env vars @@ -587,23 +530,15 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Configure CPython - run: ./configure --config-cache --with-address-sanitizer --without-pymalloc + run: ./configure --config-cache --with-address-sanitizer --without-pymalloc --with-openssl="$OPENSSL_DIR" - name: Build CPython run: make -j4 - name: Display build info @@ -615,7 +550,7 @@ jobs: # ${{ '' } is a hack to nest jobs under the same sidebar category. name: Sanitizers${{ '' }} # zizmor: ignore[obfuscation] needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' strategy: fail-fast: false matrix: @@ -633,7 +568,6 @@ jobs: uses: ./.github/workflows/reusable-san.yml with: sanitizer: ${{ matrix.sanitizer }} - config_hash: ${{ needs.build-context.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} cross-build-linux: @@ -641,18 +575,13 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Set build dir @@ -676,45 +605,49 @@ jobs: run: | "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed - # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: - name: CIFuzz - runs-on: ubuntu-latest - timeout-minutes: 60 + # ${{ '' } is a hack to nest jobs under the same sidebar category. + name: CIFuzz${{ '' }} # zizmor: ignore[obfuscation] needs: build-context - if: needs.build-context.outputs.run-ci-fuzz == 'true' + if: >- + needs.build-context.outputs.run-ci-fuzz == 'true' + || needs.build-context.outputs.run-ci-fuzz-stdlib == 'true' permissions: + contents: read security-events: write strategy: fail-fast: false matrix: - sanitizer: [address, undefined, memory] - steps: - - name: Build fuzzers (${{ matrix.sanitizer }}) - id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master - with: + sanitizer: + - address + oss-fuzz-project-name: + - cpython3 + - python3-libraries + include: + - sanitizer: undefined oss-fuzz-project-name: cpython3 - sanitizer: ${{ matrix.sanitizer }} - - name: Run fuzzers (${{ matrix.sanitizer }}) - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master - with: - fuzz-seconds: 600 + - sanitizer: memory oss-fuzz-project-name: cpython3 - output-sarif: true - sanitizer: ${{ matrix.sanitizer }} - - name: Upload crash - if: failure() && steps.build.outcome == 'success' - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.sanitizer }}-artifacts - path: ./out/artifacts - - name: Upload SARIF - if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: cifuzz-sarif/results.sarif - checkout_path: cifuzz-sarif + exclude: + # Note that the 'no-exclude' sentinel below is to prevent + # an empty string value from excluding all jobs and causing + # GHA to create a 'default' matrix entry with all empty values. + - oss-fuzz-project-name: >- + ${{ + needs.build-context.outputs.run-ci-fuzz == 'true' + && 'no-exclude' + || 'cpython3' + }} + - oss-fuzz-project-name: >- + ${{ + needs.build-context.outputs.run-ci-fuzz-stdlib == 'true' + && 'no-exclude' + || 'python3-libraries' + }} + uses: ./.github/workflows/reusable-cifuzz.yml + with: + oss-fuzz-project-name: ${{ matrix.oss-fuzz-project-name }} + sanitizer: ${{ matrix.sanitizer }} all-required-green: # This job does nothing and is only used for the branch protection name: All required checks pass @@ -725,13 +658,14 @@ jobs: - check-docs - check-autoconf-regen - check-generated-files + - check-c-api-docs - build-windows - build-windows-msi - build-macos - build-ubuntu - - build-ubuntu-ssltests-awslc - - build-ubuntu-ssltests-openssl - - build-android + - build-ubuntu-ssltests + - build-ios + - build-emscripten - build-wasi - test-hypothesis - build-asan @@ -745,30 +679,41 @@ jobs: uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe with: allowed-failures: >- + build-android, + build-emscripten, build-windows-msi, - build-ubuntu-ssltests-awslc, - build-ubuntu-ssltests-openssl, + build-ubuntu-ssltests, test-hypothesis, cifuzz, allowed-skips: >- - ${{ - !fromJSON(needs.build-context.outputs.run-docs) - && ' - check-docs, - ' - || '' - }} + ${{ !fromJSON(needs.build-context.outputs.run-docs) && 'check-docs,' || '' }} ${{ needs.build-context.outputs.run-tests != 'true' && ' check-autoconf-regen, check-generated-files, - build-macos, + ' + || '' + }} + ${{ + !fromJSON(needs.build-context.outputs.run-tests) + && !fromJSON(needs.build-context.outputs.run-docs) + && 'check-c-api-docs,' + || '' + }} + ${{ !fromJSON(needs.build-context.outputs.run-windows-tests) && 'build-windows,' || '' }} + ${{ + !fromJSON(needs.build-context.outputs.run-ci-fuzz) + && !fromJSON(needs.build-context.outputs.run-ci-fuzz-stdlib) + && 'cifuzz,' || + '' + }} + ${{ !fromJSON(needs.build-context.outputs.run-macos) && 'build-macos,' || '' }} + ${{ + !fromJSON(needs.build-context.outputs.run-ubuntu) + && ' build-ubuntu, - build-ubuntu-ssltests-awslc, - build-ubuntu-ssltests-openssl, - build-android, - build-wasi, + build-ubuntu-ssltests, test-hypothesis, build-asan, build-san, @@ -776,18 +721,8 @@ jobs: ' || '' }} - ${{ - !fromJSON(needs.build-context.outputs.run-windows-tests) - && ' - build-windows, - ' - || '' - }} - ${{ - !fromJSON(needs.build-context.outputs.run-ci-fuzz) - && ' - cifuzz, - ' - || '' - }} + ${{ !fromJSON(needs.build-context.outputs.run-android) && 'build-android,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-ios) && 'build-ios,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-emscripten) && 'build-emscripten,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-wasi) && 'build-wasi,' || '' }} jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml deleted file mode 100644 index a09a30587b3..00000000000 --- a/.github/workflows/documentation-links.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Read the Docs PR preview -# Automatically edits a pull request's descriptions with a link -# to the documentation's preview on Read the Docs. - -on: - pull_request_target: - types: - - opened - paths: - - 'Doc/**' - - '.github/workflows/doc.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - documentation-links: - runs-on: ubuntu-latest - permissions: - pull-requests: write - timeout-minutes: 5 - - steps: - - uses: readthedocs/actions/preview@v1 - with: - project-slug: "cpython-previews" - single-version: "true" diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 51a069d857f..e63fe9e1284 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -1,25 +1,18 @@ name: JIT on: pull_request: - paths: + paths: &paths - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' - 'Python/executor_cases.c.h' - 'Python/optimizer_cases.c.h' + - '**_testinternalcapi**' - '!Python/perf_jit_trampoline.c' - '!**/*.md' - '!**/*.ini' push: - paths: - - '**jit**' - - 'Python/bytecodes.c' - - 'Python/optimizer*.c' - - 'Python/executor_cases.c.h' - - 'Python/optimizer_cases.c.h' - - '!Python/perf_jit_trampoline.c' - - '!**/*.md' - - '!**/*.ini' + paths: *paths workflow_dispatch: permissions: @@ -31,14 +24,15 @@ concurrency: env: FORCE_COLOR: 1 + LLVM_VERSION: 21 jobs: interpreter: name: Interpreter (Debug) runs-on: ubuntu-24.04 - timeout-minutes: 90 + timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Build tier two interpreter @@ -48,11 +42,12 @@ jobs: - name: Test tier two interpreter run: | ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - jit: + + windows: name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) - needs: interpreter + runs-on: ${{ matrix.runner }} - timeout-minutes: 90 + timeout-minutes: 60 strategy: fail-fast: false matrix: @@ -60,62 +55,66 @@ jobs: - i686-pc-windows-msvc/msvc - x86_64-pc-windows-msvc/msvc - aarch64-pc-windows-msvc/msvc - - x86_64-apple-darwin/clang - - aarch64-apple-darwin/clang - - x86_64-unknown-linux-gnu/gcc - - aarch64-unknown-linux-gnu/gcc debug: - true - false - llvm: - - 19 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 - runner: windows-latest + runner: windows-2025-vs2026 - target: x86_64-pc-windows-msvc/msvc architecture: x64 - runner: windows-latest + runner: windows-2025-vs2026 - target: aarch64-pc-windows-msvc/msvc architecture: ARM64 runner: windows-11-arm - - target: x86_64-apple-darwin/clang - architecture: x86_64 - runner: macos-13 - - target: aarch64-apple-darwin/clang - architecture: aarch64 - runner: macos-14 - - target: x86_64-unknown-linux-gnu/gcc - architecture: x86_64 - runner: ubuntu-24.04 - - target: aarch64-unknown-linux-gnu/gcc - architecture: aarch64 - runner: ubuntu-24.04-arm steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' - # PCbuild downloads LLVM automatically: - - name: Windows - if: runner.os == 'Windows' + - name: Build run: | ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} + - name: Test + run: | ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. - # This is a bug in the macOS runner image where the pre-installed Python is installed in the same - # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes - # the symlink to the pre-installed Python so that the Homebrew Python is used instead. - - name: macOS - if: runner.os == 'macOS' + macos: + name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) + + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + target: + - x86_64-apple-darwin/clang + - aarch64-apple-darwin/clang + debug: + - true + - false + include: + - target: x86_64-apple-darwin/clang + runner: macos-15-intel + - target: aarch64-apple-darwin/clang + runner: macos-15 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - name: Install LLVM run: | brew update - find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete - brew install llvm@${{ matrix.llvm }} + brew install llvm@${{ env.LLVM_VERSION }} + - name: Build + run: | export SDKROOT="$(xcrun --show-sdk-path)" # Set MACOSX_DEPLOYMENT_TARGET and -Werror=unguarded-availability to # make sure we don't break downstream distributors (like uv): @@ -123,41 +122,83 @@ jobs: export MACOSX_DEPLOYMENT_TARGET=10.15 ./configure --enable-experimental-jit --enable-universalsdk --with-universal-archs=universal2 ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 + - name: Test + run: | ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - name: Linux - if: runner.os == 'Linux' + linux: + name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) + + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + target: + - x86_64-unknown-linux-gnu/gcc + - aarch64-unknown-linux-gnu/gcc + debug: + - true + - false + include: + - target: x86_64-unknown-linux-gnu/gcc + runner: ubuntu-24.04 + - target: aarch64-unknown-linux-gnu/gcc + runner: ubuntu-24.04-arm + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - name: Build run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ env.LLVM_VERSION }} + export PATH="$(llvm-config-${{ env.LLVM_VERSION }} --bindir):$PATH" ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 + - name: Test + run: | ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - # XXX: GH-133171 - # jit-with-disabled-gil: - # name: Free-Threaded (Debug) - # needs: interpreter - # runs-on: ubuntu-24.04 - # timeout-minutes: 90 - # strategy: - # fail-fast: false - # matrix: - # llvm: - # - 19 - # steps: - # - uses: actions/checkout@v4 - # with: - # persist-credentials: false - # - uses: actions/setup-python@v5 - # with: - # python-version: '3.11' - # - name: Build with JIT enabled and GIL disabled - # run: | - # sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - # export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - # ./configure --enable-experimental-jit --with-pydebug --disable-gil - # make all --jobs 4 - # - name: Run tests - # run: | - # ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + linux-extras: + name: ${{ matrix.name }} + + runs-on: ubuntu-24.04 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - name: Free-Threaded (Debug) + configure_flags: --enable-experimental-jit --with-pydebug --disable-gil + continue_on_error: true + - name: JIT without optimizations (Debug) + configure_flags: --enable-experimental-jit --with-pydebug + test_env: "PYTHON_UOPS_OPTIMIZE=0" + - name: JIT with tail calling interpreter + configure_flags: --enable-experimental-jit --with-tail-call-interp --with-pydebug + use_clang: true + run_tests: false + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - name: Build + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ env.LLVM_VERSION }} + export PATH="$(llvm-config-${{ env.LLVM_VERSION }} --bindir):$PATH" + if [ "${{ matrix.use_clang }}" = "true" ]; then + export CC=clang-${{ env.LLVM_VERSION }} + fi + ./configure ${{ matrix.configure_flags }} + make all --jobs 4 + - name: Test + if: matrix.run_tests != false + run: | + ${{ matrix.test_env }} ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + continue-on-error: ${{ matrix.continue_on_error }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d74ce8fcc25..e9a4eb2b080 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,10 +19,7 @@ jobs: timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - uses: pre-commit/action@v3.0.1 + - uses: j178/prek-action@0bb87d7f00b0c99306c8bcb8b8beba1eb581c037 # v1.1.1 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 5d5d77f29f6..7f6571ef954 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -16,8 +16,10 @@ on: - "Tools/build/check_extension_modules.py" - "Tools/build/check_warnings.py" - "Tools/build/compute-changes.py" + - "Tools/build/consts_getter.py" - "Tools/build/deepfreeze.py" - "Tools/build/generate-build-details.py" + - "Tools/build/generate_levenshtein_examples.py" - "Tools/build/generate_sbom.py" - "Tools/build/generate_stdlib_module_names.py" - "Tools/build/mypy.ini" @@ -25,6 +27,7 @@ on: - "Tools/build/update_file.py" - "Tools/build/verify_ensurepip_wheels.py" - "Tools/cases_generator/**" + - "Tools/check-c-api-docs/**" - "Tools/clinic/**" - "Tools/jit/**" - "Tools/peg_generator/**" @@ -57,15 +60,16 @@ jobs: "Lib/tomllib", "Tools/build", "Tools/cases_generator", + "Tools/check-c-api-docs", "Tools/clinic", "Tools/jit", "Tools/peg_generator", ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.13" cache: pip diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml index 9f1a8a824e5..1267361040c 100644 --- a/.github/workflows/new-bugs-announce-notifier.yml +++ b/.github/workflows/new-bugs-announce-notifier.yml @@ -6,19 +6,21 @@ on: - opened permissions: - issues: read + contents: read jobs: notify-new-bugs-announce: runs-on: ubuntu-latest + permissions: + issues: read timeout-minutes: 10 steps: - - uses: actions/setup-node@v4 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: 20 - run: npm install mailgun.js form-data - name: Send notification - uses: actions/github-script@v7 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: MAILGUN_API_KEY: ${{ secrets.MAILGUN_PYTHON_ORG_MAILGUN_KEY }} with: @@ -44,7 +46,7 @@ jobs: // We need to truncate the body size, because the max size for // the whole payload is 16kb. We want to be safe and assume that // body can take up to ~8kb of space. - body : issue.data.body.substring(0, 8000) + body : (issue.data.body || "").substring(0, 8000) }; const data = { diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index 0b64367e6c4..6201e719ca8 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -1,10 +1,9 @@ #!/bin/sh apt-get update -apt-get -yq install \ +apt-get -yq --no-install-recommends install \ build-essential \ pkg-config \ - ccache \ cmake \ gdb \ lcov \ @@ -27,9 +26,16 @@ apt-get -yq install \ xvfb \ zlib1g-dev -# Workaround missing libmpdec-dev on ubuntu 24.04: -# https://launchpad.net/~ondrej/+archive/ubuntu/php -# https://deb.sury.org/ -sudo add-apt-repository ppa:ondrej/php -apt-get update -apt-get -yq install libmpdec-dev +# Workaround missing libmpdec-dev on ubuntu 24.04 by building mpdecimal +# from source. ppa:ondrej/php (launchpad.net) are unreliable +# (https://status.canonical.com) so fetch the tarball directly +# from the upstream host. +# https://www.bytereef.org/mpdecimal/ +MPDECIMAL_VERSION=4.0.1 +curl -fsSL "https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-${MPDECIMAL_VERSION}.tar.gz" \ + | tar -xz -C /tmp +(cd "/tmp/mpdecimal-${MPDECIMAL_VERSION}" \ + && ./configure --prefix=/usr/local \ + && make -j"$(nproc)" \ + && make install) +ldconfig diff --git a/.github/workflows/project-updater.yml b/.github/workflows/project-updater.yml deleted file mode 100644 index 1d9d637ec84..00000000000 --- a/.github/workflows/project-updater.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Update GH projects - -on: - issues: - types: - - opened - - labeled - -permissions: - contents: read - -jobs: - add-to-project: - name: Add issues to projects - runs-on: ubuntu-latest - timeout-minutes: 10 - strategy: - fail-fast: false - matrix: - include: - # if an issue has any of these labels, it will be added - # to the corresponding project - - { project: 2, label: "release-blocker, deferred-blocker" } - - { project: 32, label: sprint } - - steps: - - uses: actions/add-to-project@v1.0.0 - with: - project-url: https://github.com/orgs/python/projects/${{ matrix.project }} - github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} - labeled: ${{ matrix.label }} diff --git a/.github/workflows/regen-abidump.sh b/.github/workflows/regen-abidump.sh index 251bb3857ec..75a1a72e370 100644 --- a/.github/workflows/regen-abidump.sh +++ b/.github/workflows/regen-abidump.sh @@ -2,7 +2,7 @@ set -ex export DEBIAN_FRONTEND=noninteractive ./.github/workflows/posix-deps-apt.sh -apt-get install -yq abigail-tools python3 +apt-get install -yq --no-install-recommends abigail-tools python3 export CFLAGS="-g3 -O0" ./configure --enable-shared && make make regen-abidump diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index 7e534c58c79..f3e26668795 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -4,6 +4,9 @@ on: pull_request: types: [opened, reopened, labeled, unlabeled, synchronize] +permissions: + contents: read + jobs: label-dnm: name: DO-NOT-MERGE @@ -15,7 +18,7 @@ jobs: steps: - name: Check there's no DO-NOT-MERGE - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 0 @@ -33,7 +36,7 @@ jobs: steps: # Check that the PR is not awaiting changes from the author due to previous review. - name: Check there's no required changes - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 0 @@ -42,7 +45,7 @@ jobs: awaiting change review - id: is-feature name: Check whether this PR is a feature (contains a "type-feature" label) - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 1 @@ -53,7 +56,7 @@ jobs: - id: awaiting-merge if: steps.is-feature.outputs.status == 'success' name: Check for complete review - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 1 diff --git a/.github/workflows/reusable-check-c-api-docs.yml b/.github/workflows/reusable-check-c-api-docs.yml new file mode 100644 index 00000000000..49e5ef7f768 --- /dev/null +++ b/.github/workflows/reusable-check-c-api-docs.yml @@ -0,0 +1,25 @@ +name: Reusable C API Docs Check + +on: + workflow_call: + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + +jobs: + check-c-api-docs: + name: 'Check if all C APIs are documented' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.x' + - name: Check for undocumented C APIs + run: python Tools/check-c-api-docs/main.py diff --git a/.github/workflows/reusable-check-html-ids.yml b/.github/workflows/reusable-check-html-ids.yml new file mode 100644 index 00000000000..4f827c55cac --- /dev/null +++ b/.github/workflows/reusable-check-html-ids.yml @@ -0,0 +1,67 @@ +name: Reusable check HTML IDs + +on: + workflow_call: + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + +jobs: + check-html-ids: + name: 'Check for removed HTML IDs' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: 'Check out PR head' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha }} + - name: 'Find merge base' + id: merge-base + run: | + BASE="${{ github.event.pull_request.base.sha }}" + HEAD="${{ github.event.pull_request.head.sha }}" + git fetch --depth=$((${{ github.event.pull_request.commits }} + 10)) --no-tags origin "$BASE" "$HEAD" + + if ! MERGE_BASE=$(git merge-base "$BASE" "$HEAD" 2>/dev/null); then + git fetch --deepen=1 --no-tags origin "$BASE" "$HEAD" + + OLDEST=$(git rev-list --reflog --max-parents=0 --reverse "${BASE}^" "${HEAD}^" | head -1) + TIMESTAMP=$(git show --format=%at --no-patch "$OLDEST") + + git fetch --shallow-since="$TIMESTAMP" --no-tags origin "$BASE" "$HEAD" + + MERGE_BASE=$(git merge-base "$BASE" "$HEAD") + fi + echo "sha=$MERGE_BASE" >> "$GITHUB_OUTPUT" + - name: 'Create worktree at merge base' + env: + MERGE_BASE: ${{ steps.merge-base.outputs.sha }} + run: git worktree add /tmp/merge-base "$MERGE_BASE" --detach + - name: 'Set up Python' + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3' + cache: 'pip' + cache-dependency-path: 'Doc/requirements.txt' + - name: 'Install build dependencies' + run: make -C /tmp/merge-base/Doc/ venv + - name: 'Build HTML documentation' + run: make -C /tmp/merge-base/Doc/ SPHINXOPTS="--quiet" html + - name: 'Collect HTML IDs' + run: python Doc/tools/check-html-ids.py collect /tmp/merge-base/Doc/build/html -o /tmp/html-ids-base.json.gz + - name: 'Download PR head HTML IDs' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: html-ids-head.json.gz + path: /tmp + - name: 'Check for removed HTML IDs' + run: | + # shellcheck disable=SC2046 + python Doc/tools/check-html-ids.py -v check \ + /tmp/html-ids-base.json.gz /tmp/html-ids-head.json.gz \ + $([ -f Doc/tools/removed-ids.txt ] && echo "--exclude-file Doc/tools/removed-ids.txt") diff --git a/.github/workflows/reusable-cifuzz.yml b/.github/workflows/reusable-cifuzz.yml new file mode 100644 index 00000000000..0d022326863 --- /dev/null +++ b/.github/workflows/reusable-cifuzz.yml @@ -0,0 +1,49 @@ +# CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ +name: Reusable CIFuzz + +on: + workflow_call: + inputs: + oss-fuzz-project-name: + description: OSS-Fuzz project name + required: true + type: string + sanitizer: + description: OSS-Fuzz sanitizer + required: true + type: string + +permissions: + contents: read + +jobs: + cifuzz: + name: ${{ inputs.oss-fuzz-project-name }} (${{ inputs.sanitizer }}) + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Build fuzzers (${{ inputs.sanitizer }}) + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@ed23f8af80ff82b25ca67cd9b101e690b8897b3f # master + with: + oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }} + sanitizer: ${{ inputs.sanitizer }} + - name: Run fuzzers (${{ inputs.sanitizer }}) + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@ed23f8af80ff82b25ca67cd9b101e690b8897b3f # master + with: + fuzz-seconds: 600 + oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }} + output-sarif: true + sanitizer: ${{ inputs.sanitizer }} + - name: Upload crash + if: failure() && steps.build.outcome == 'success' + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: ${{ inputs.sanitizer }}-artifacts + path: ./out/artifacts + - name: Upload SARIF + if: always() && steps.build.outcome == 'success' + uses: github/codeql-action/upload-sarif@38697555549f1db7851b81482ff19f1fa5c4fedc # v4.34.1 + with: + sarif_file: cifuzz-sarif/results.sarif + checkout_path: cifuzz-sarif diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index d2668ddcac1..b8a9e2960ec 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -17,24 +17,45 @@ on: # yamllint disable-line rule:truthy # || 'falsy-branch' # }} # - config-hash: - description: Config hash value for use in cache keys - value: ${{ jobs.compute-changes.outputs.config-hash }} # str + run-android: + description: Whether to run the Android tests + value: ${{ jobs.compute-changes.outputs.run-android }} # bool + run-ci-fuzz: + description: Whether to run the CIFuzz job for 'cpython' fuzzer + value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }} # bool + run-ci-fuzz-stdlib: + description: Whether to run the CIFuzz job for 'python3-libraries' fuzzer + value: ${{ jobs.compute-changes.outputs.run-ci-fuzz-stdlib }} # bool run-docs: description: Whether to build the docs value: ${{ jobs.compute-changes.outputs.run-docs }} # bool + run-ios: + description: Whether to run the iOS tests + value: ${{ jobs.compute-changes.outputs.run-ios }} # bool + run-macos: + description: Whether to run the macOS tests + value: ${{ jobs.compute-changes.outputs.run-macos }} # bool run-tests: description: Whether to run the regular tests value: ${{ jobs.compute-changes.outputs.run-tests }} # bool - run-windows-tests: - description: Whether to run the Windows tests - value: ${{ jobs.compute-changes.outputs.run-windows-tests }} # bool + run-ubuntu: + description: Whether to run the Ubuntu tests + value: ${{ jobs.compute-changes.outputs.run-ubuntu }} # bool + run-emscripten: + description: Whether to run the Emscripten tests + value: ${{ jobs.compute-changes.outputs.run-emscripten }} # bool + run-wasi: + description: Whether to run the WASI tests + value: ${{ jobs.compute-changes.outputs.run-wasi }} # bool run-windows-msi: description: Whether to run the MSI installer smoke tests value: ${{ jobs.compute-changes.outputs.run-windows-msi }} # bool - run-ci-fuzz: - description: Whether to run the CIFuzz job - value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }} # bool + run-windows-tests: + description: Whether to run the Windows tests + value: ${{ jobs.compute-changes.outputs.run-windows-tests }} # bool + +permissions: + contents: read jobs: compute-changes: @@ -42,22 +63,28 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 outputs: - config-hash: ${{ steps.config-hash.outputs.hash }} + run-android: ${{ steps.changes.outputs.run-android }} run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }} + run-ci-fuzz-stdlib: ${{ steps.changes.outputs.run-ci-fuzz-stdlib }} run-docs: ${{ steps.changes.outputs.run-docs }} + run-ios: ${{ steps.changes.outputs.run-ios }} + run-macos: ${{ steps.changes.outputs.run-macos }} run-tests: ${{ steps.changes.outputs.run-tests }} + run-ubuntu: ${{ steps.changes.outputs.run-ubuntu }} + run-emscripten: ${{ steps.changes.outputs.run-emscripten }} + run-wasi: ${{ steps.changes.outputs.run-wasi }} run-windows-msi: ${{ steps.changes.outputs.run-windows-msi }} run-windows-tests: ${{ steps.changes.outputs.run-windows-tests }} steps: - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3" - run: >- echo '${{ github.event_name }}' - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false ref: >- @@ -100,8 +127,3 @@ jobs: GITHUB_EVENT_NAME: ${{ github.event_name }} CCF_TARGET_REF: ${{ github.base_ref || github.event.repository.default_branch }} CCF_HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }} - - - name: Compute hash for config cache key - id: config-hash - run: | - echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 65154aae4c4..7b524569f85 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -27,7 +27,7 @@ jobs: refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' steps: - name: 'Check out latest PR branch commit' - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false ref: >- @@ -52,11 +52,11 @@ jobs: git fetch origin "${refspec_base}" --shallow-since="${DATE}" \ --no-tags --prune --no-recurse-submodules - name: 'Set up Python' - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3' cache: 'pip' - cache-dependency-path: 'Doc/requirements.txt' + cache-dependency-path: 'Doc/pylock.toml' - name: 'Install build dependencies' run: make -C Doc/ venv @@ -75,6 +75,22 @@ jobs: --fail-if-regression \ --fail-if-improved \ --fail-if-new-news-nit + - name: 'Collect HTML IDs' + if: github.event_name == 'pull_request' + run: python Doc/tools/check-html-ids.py collect Doc/build/html -o Doc/build/html-ids-head.json.gz + - name: 'Upload HTML IDs' + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: html-ids-head + path: Doc/build/html-ids-head.json.gz + archive: false + + check-html-ids: + name: 'Check for removed HTML IDs' + needs: build-doc + if: github.event_name == 'pull_request' + uses: ./.github/workflows/reusable-check-html-ids.yml # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: @@ -82,17 +98,17 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/cache@v4 + - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ~/.cache/pip key: ubuntu-doc-${{ hashFiles('Doc/requirements.txt') }} restore-keys: | ubuntu-doc- - name: 'Install Dependencies' - run: sudo ./.github/workflows/posix-deps-apt.sh && sudo apt-get install wamerican + run: sudo ./.github/workflows/posix-deps-apt.sh && sudo apt-get install --no-install-recommends wamerican - name: 'Configure CPython' run: ./configure --with-pydebug - name: 'Build CPython' @@ -108,11 +124,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: 'Set up Python' - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3' cache: 'pip' diff --git a/.github/workflows/reusable-emscripten.yml b/.github/workflows/reusable-emscripten.yml new file mode 100644 index 00000000000..69a780a9aeb --- /dev/null +++ b/.github/workflows/reusable-emscripten.yml @@ -0,0 +1,77 @@ +name: Reusable Emscripten + +on: + workflow_call: + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + +jobs: + build-emscripten-reusable: + name: 'build and test' + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: "Read Emscripten config" + id: emscripten-config + shell: python + run: | + import hashlib + import json + import os + import tomllib + from pathlib import Path + + config = tomllib.loads(Path("Platforms/emscripten/config.toml").read_text()) + h = hashlib.sha256() + h.update(json.dumps(config["dependencies"], sort_keys=True).encode()) + h.update(Path("Platforms/emscripten/make_libffi.sh").read_bytes()) + h.update(b'1') # Update to explicitly bust cache + emsdk_cache = Path(os.environ["RUNNER_TEMP"]) / "emsdk-cache" + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"emscripten-version={config['emscripten-version']}\n") + f.write(f"node-version={config['node-version']}\n") + f.write(f"deps-hash={h.hexdigest()}\n") + with open(os.environ["GITHUB_ENV"], "a") as f: + f.write(f"EMSDK_CACHE={emsdk_cache}\n") + - name: "Install Node.js" + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: ${{ steps.emscripten-config.outputs.node-version }} + - name: "Cache Emscripten SDK" + id: emsdk-cache + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: ${{ env.EMSDK_CACHE }} + key: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }}-${{ steps.emscripten-config.outputs.deps-hash }} + restore-keys: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }} + - name: "Install Python" + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.x' + - name: "Runner image version" + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: "Install Emscripten" + run: python3 Platforms/emscripten install-emscripten + - name: "Configure build Python" + run: python3 Platforms/emscripten configure-build-python -- --config-cache --with-pydebug + - name: "Make build Python" + run: python3 Platforms/emscripten make-build-python + - name: "Make dependencies" + run: >- + python3 Platforms/emscripten make-dependencies + ${{ steps.emsdk-cache.outputs.cache-hit == 'true' && '--check-up-to-date' || '' }} + - name: "Configure host Python" + run: python3 Platforms/emscripten configure-host --host-runner node -- --config-cache + - name: "Make host Python" + run: python3 Platforms/emscripten make-host + - name: "Display build info" + run: python3 Platforms/emscripten run --pythoninfo + - name: "Test" + run: python3 Platforms/emscripten run --test diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index de0c4022136..f10503055b2 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -3,9 +3,6 @@ name: Reusable macOS on: workflow_call: inputs: - config_hash: - required: true - type: string free-threading: required: false type: boolean @@ -15,6 +12,9 @@ on: required: true type: string +permissions: + contents: read + env: FORCE_COLOR: 1 @@ -31,21 +31,16 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - name: Install Homebrew dependencies run: | - brew install pkg-config openssl@3.0 xz gdbm tcl-tk@8 make + brew install pkg-config openssl@3.5 xz gdbm tcl-tk@9 make # Because alternate versions are not symlinked into place by default: - brew link --overwrite tcl-tk@8 + brew link --overwrite tcl-tk@9 - name: Configure CPython run: | MACOSX_DEPLOYMENT_TARGET=10.15 \ @@ -58,17 +53,17 @@ jobs: --enable-safety \ ${{ inputs.free-threading && '--disable-gil' || '' }} \ --prefix=/opt/python-dev \ - --with-openssl="$(brew --prefix openssl@3.0)" + --with-openssl="$(brew --prefix openssl@3.5)" - name: Build CPython - if : ${{ inputs.free-threading || inputs.os != 'macos-13' }} + if : ${{ inputs.free-threading || inputs.os != 'macos-15-intel' }} run: gmake -j8 - name: Build CPython for compiler warning check - if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + if : ${{ !inputs.free-threading && inputs.os == 'macos-15-intel' }} run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt - name: Display build info run: make pythoninfo - name: Check compiler warnings - if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + if : ${{ !inputs.free-threading && inputs.os == 'macos-15-intel' }} run: >- python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output_macos.txt diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index e6ff02e4838..33f6f0ef455 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -6,15 +6,15 @@ on: sanitizer: required: true type: string - config_hash: - required: true - type: string free-threading: description: Whether to use free-threaded mode required: false type: boolean default: false +permissions: + contents: read + env: FORCE_COLOR: 1 @@ -29,33 +29,26 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.sanitizer }}-${{ inputs.config_hash }} - name: Install dependencies run: | sudo ./.github/workflows/posix-deps-apt.sh # Install clang wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh + sudo ./llvm.sh 20 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-20 100 + sudo update-alternatives --set clang /usr/bin/clang-20 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-20 100 + sudo update-alternatives --set clang++ /usr/bin/clang++-20 if [ "${SANITIZER}" = "TSan" ]; then - sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 - sudo update-alternatives --set clang /usr/bin/clang-17 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 - sudo update-alternatives --set clang++ /usr/bin/clang++-17 # Reduce ASLR to avoid TSan crashing sudo sysctl -w vm.mmap_rnd_bits=28 - else - sudo ./llvm.sh 20 fi - name: Sanitizer option setup @@ -67,21 +60,13 @@ jobs: || '' }}.txt handle_segv=0" >> "$GITHUB_ENV" else - echo "UBSAN_OPTIONS=${SAN_LOG_OPTION}" >> "$GITHUB_ENV" + echo "UBSAN_OPTIONS=${SAN_LOG_OPTION} halt_on_error=1 suppressions=${GITHUB_WORKSPACE}/Tools/ubsan/suppressions.txt" >> "$GITHUB_ENV" fi echo "CC=clang" >> "$GITHUB_ENV" echo "CXX=clang++" >> "$GITHUB_ENV" env: SANITIZER: ${{ inputs.sanitizer }} SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Configure CPython run: >- ./configure @@ -89,7 +74,7 @@ jobs: ${{ inputs.sanitizer == 'TSan' && '--with-thread-sanitizer' - || '--with-undefined-behavior-sanitizer' + || '--with-undefined-behavior-sanitizer --with-strict-overflow' }} --with-pydebug ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} @@ -97,10 +82,13 @@ jobs: run: make -j4 - name: Display build info run: make pythoninfo + # test_{capi,faulthandler} are skipped under UBSan because + # they raise signals that UBSan with halt_on_error=1 intercepts. - name: Tests run: >- ./python -m test ${{ inputs.sanitizer == 'TSan' && '--tsan' || '' }} + ${{ inputs.sanitizer == 'UBSan' && '-x test_capi -x test_faulthandler' || '' }} -j4 - name: Parallel tests if: >- @@ -112,7 +100,7 @@ jobs: run: find "${GITHUB_WORKSPACE}" -name 'san_log.*' | xargs head -n 1000 - name: Archive logs if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: >- ${{ inputs.sanitizer }}-logs-${{ diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 76b19fd5d1a..87fba6221fb 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -3,9 +3,6 @@ name: Reusable Ubuntu on: workflow_call: inputs: - config_hash: - required: true - type: string bolt-optimizations: description: Whether to enable BOLT optimizations required: false @@ -20,6 +17,14 @@ on: description: OS to run the job required: true type: string + test-opts: + description: Extra options to pass to the test runner via TESTOPTS + required: false + type: string + default: '' + +permissions: + contents: read env: FORCE_COLOR: 1 @@ -30,11 +35,11 @@ jobs: runs-on: ${{ inputs.os }} timeout-minutes: 60 env: - OPENSSL_VER: 3.0.15 + OPENSSL_VER: 3.5.5 PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Register gcc problem matcher @@ -45,7 +50,7 @@ jobs: if: ${{ fromJSON(inputs.bolt-optimizations) }} run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh 19 - sudo apt-get install bolt-19 + sudo apt-get install --no-install-recommends bolt-19 echo PATH="$(llvm-config-19 --bindir):$PATH" >> $GITHUB_ENV - name: Configure OpenSSL env vars run: | @@ -54,21 +59,13 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ inputs.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" @@ -79,11 +76,6 @@ jobs: run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} # `test_unpickle_module_race` writes to the source directory, which is @@ -124,4 +116,6 @@ jobs: run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw - name: Tests working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: xvfb-run make ci + run: xvfb-run make ci EXTRATESTOPTS="${TEST_OPTS}" + env: + TEST_OPTS: ${{ inputs.test-opts }} diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 6beb91e66d4..48fb70cbff8 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -2,10 +2,9 @@ name: Reusable WASI on: workflow_call: - inputs: - config_hash: - required: true - type: string + +permissions: + contents: read env: FORCE_COLOR: 1 @@ -13,71 +12,55 @@ env: jobs: build-wasi-reusable: name: 'build and test' - runs-on: ubuntu-24.04 + runs-on: ubuntu-24.04-arm timeout-minutes: 60 env: - WASMTIME_VERSION: 22.0.0 - WASI_SDK_VERSION: 24 - WASI_SDK_PATH: /opt/wasi-sdk + WASMTIME_VERSION: 38.0.3 CROSS_BUILD_PYTHON: cross-build/build CROSS_BUILD_WASI: cross-build/wasm32-wasip1 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false # No problem resolver registered as one doesn't currently exist for Clang. - name: "Install wasmtime" - uses: bytecodealliance/actions/wasmtime/setup@v1 + uses: bytecodealliance/actions/wasmtime/setup@9152e710e9f7182e4c29ad218e4f335a7b203613 # v1.1.3 with: version: ${{ env.WASMTIME_VERSION }} - - name: "Restore WASI SDK" - id: cache-wasi-sdk - uses: actions/cache@v4 - with: - path: ${{ env.WASI_SDK_PATH }} - key: ${{ runner.os }}-wasi-sdk-${{ env.WASI_SDK_VERSION }} - - name: "Install WASI SDK" # Hard-coded to x64. - if: steps.cache-wasi-sdk.outputs.cache-hit != 'true' + - name: "Read WASI SDK version" + id: wasi-sdk-version run: | - mkdir "${WASI_SDK_PATH}" && \ - curl -s -S --location "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz" | \ - tar --strip-components 1 --directory "${WASI_SDK_PATH}" --extract --gunzip - - name: "Configure ccache action" - uses: hendrikmuhs/ccache-action@v1.2 + import tomllib + from pathlib import Path + import os + config = tomllib.loads(Path("Platforms/WASI/config.toml").read_text()) + version = config["targets"]["wasi-sdk"] + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"version={version}\n") + shell: python + - name: "Install WASI SDK" + id: install-wasi-sdk + uses: bytecodealliance/setup-wasi-sdk-action@b2de090b44eb70013ee96b393727d473b35e1728 with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - - name: "Add ccache to PATH" - run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + version: ${{ steps.wasi-sdk-version.outputs.version }} + add-to-path: false - name: "Install Python" - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.x' - name: "Runner image version" run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: "Restore Python build config.cache" - uses: actions/cache@v4 - with: - path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache - # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python. - # Include the hash of `Tools/wasm/wasi.py` as it may change the environment variables. - # (Make sure to keep the key in sync with the other config.cache step below.) - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} - name: "Configure build Python" - run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug + run: python3 Platforms/WASI configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" - run: python3 Tools/wasm/wasi.py make-build-python - - name: "Restore host config.cache" - uses: actions/cache@v4 - with: - path: ${{ env.CROSS_BUILD_WASI }}/config.cache - # Should be kept in sync with the other config.cache step above. - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} + run: python3 Platforms/WASI make-build-python - name: "Configure host" # `--with-pydebug` inferred from configure-build-python - run: python3 Tools/wasm/wasi.py configure-host -- --config-cache + run: python3 Platforms/WASI configure-host -- --config-cache + env: + WASI_SDK_PATH: ${{ steps.install-wasi-sdk.outputs.wasi-sdk-path }} - name: "Make host" - run: python3 Tools/wasm/wasi.py make-host + run: python3 Platforms/WASI make-host - name: "Display build info" run: make --directory "${CROSS_BUILD_WASI}" pythoninfo - name: "Test" diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml index a50de344bba..a74724323ec 100644 --- a/.github/workflows/reusable-windows-msi.yml +++ b/.github/workflows/reusable-windows-msi.yml @@ -17,13 +17,13 @@ env: jobs: build: name: installer for ${{ inputs.arch }} - runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-latest' }} + runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2025-vs2026' }} timeout-minutes: 60 env: ARCH: ${{ inputs.arch }} IncludeFreethreaded: true steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Build CPython installer diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 37c802095b0..4c8d0c8a2f9 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -12,6 +12,13 @@ on: required: false type: boolean default: false + interpreter: + description: Which interpreter to build (switch-case or tail-call) + required: true + type: string + +permissions: + contents: read env: FORCE_COLOR: 1 @@ -20,22 +27,25 @@ env: jobs: build: - name: Build and test (${{ inputs.arch }}) - runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-latest' }} + name: Build and test (${{ inputs.arch }}, ${{ inputs.interpreter }}) + runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2025-vs2026' }} timeout-minutes: 60 env: ARCH: ${{ inputs.arch }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Register MSVC problem matcher if: inputs.arch != 'Win32' run: echo "::add-matcher::.github/problem-matchers/msvc.json" - name: Build CPython + # msvc::musttail is not supported for debug builds, so we have to + # switch to release. run: >- .\\PCbuild\\build.bat - -e -d -v + -e -v + ${{ inputs.interpreter == 'switch-case' && '-d' || '--tail-call-interp -c Release' }} -p "${ARCH}" ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} shell: bash @@ -45,6 +55,7 @@ jobs: run: >- .\\PCbuild\\rt.bat -p "${ARCH}" - -d -q --fast-ci + -q --fast-ci + ${{ inputs.interpreter == 'switch-case' && '-d' || '' }} ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} shell: bash diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index febb2dd823a..01fe5ba8fda 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -4,17 +4,21 @@ on: schedule: - cron: "0 */6 * * *" +permissions: + contents: read + jobs: stale: if: github.repository_owner == 'python' runs-on: ubuntu-latest permissions: + actions: write pull-requests: write timeout-minutes: 10 steps: - name: "Check PRs" - uses: actions/stale@v9 + uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.' diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index 57c92e193a9..656a14906b3 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -1,19 +1,14 @@ name: Tail calling interpreter on: pull_request: - paths: + paths: &paths - '.github/workflows/tail-call.yml' - 'Python/bytecodes.c' - 'Python/ceval.c' - 'Python/ceval_macros.h' - 'Python/generated_cases.c.h' push: - paths: - - '.github/workflows/tail-call.yml' - - 'Python/bytecodes.c' - - 'Python/ceval.c' - - 'Python/ceval_macros.h' - - 'Python/generated_cases.c.h' + paths: *paths workflow_dispatch: permissions: @@ -25,115 +20,73 @@ concurrency: env: FORCE_COLOR: 1 + LLVM_VERSION: 21 jobs: - tail-call: + macos: name: ${{ matrix.target }} runs-on: ${{ matrix.runner }} - timeout-minutes: 90 + timeout-minutes: 60 strategy: fail-fast: false matrix: - target: -# Un-comment as we add support for more platforms for tail-calling interpreters. -# - i686-pc-windows-msvc/msvc - - x86_64-pc-windows-msvc/msvc -# - aarch64-pc-windows-msvc/msvc - - x86_64-apple-darwin/clang - - aarch64-apple-darwin/clang - - x86_64-unknown-linux-gnu/gcc - - aarch64-unknown-linux-gnu/gcc - - free-threading - llvm: - - 20 include: -# - target: i686-pc-windows-msvc/msvc -# architecture: Win32 -# runner: windows-latest - - target: x86_64-pc-windows-msvc/msvc - architecture: x64 - runner: windows-latest -# - target: aarch64-pc-windows-msvc/msvc -# architecture: ARM64 -# runner: windows-latest - target: x86_64-apple-darwin/clang - architecture: x86_64 - runner: macos-13 + runner: macos-15-intel - target: aarch64-apple-darwin/clang - architecture: aarch64 - runner: macos-14 - - target: x86_64-unknown-linux-gnu/gcc - architecture: x86_64 - runner: ubuntu-24.04 - - target: aarch64-unknown-linux-gnu/gcc - architecture: aarch64 - runner: ubuntu-24.04-arm - - target: free-threading - architecture: x86_64 - runner: ubuntu-24.04 + runner: macos-15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' - - - name: Native Windows (debug) - if: runner.os == 'Windows' && matrix.architecture != 'ARM64' - shell: cmd - run: | - choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 - set PlatformToolset=clangcl - set LLVMToolsVersion=${{ matrix.llvm }}.1.0 - set LLVMInstallDir=C:\Program Files\LLVM - call ./PCbuild/build.bat --tail-call-interp -d -p ${{ matrix.architecture }} - call ./PCbuild/rt.bat -d -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - # No tests (yet): - - name: Emulated Windows (release) - if: runner.os == 'Windows' && matrix.architecture == 'ARM64' - shell: cmd - run: | - choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 - set PlatformToolset=clangcl - set LLVMToolsVersion=${{ matrix.llvm }}.1.0 - set LLVMInstallDir=C:\Program Files\LLVM - ./PCbuild/build.bat --tail-call-interp -p ${{ matrix.architecture }} - - # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. - # This is a bug in the macOS runner image where the pre-installed Python is installed in the same - # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes - # the symlink to the pre-installed Python so that the Homebrew Python is used instead. - # Note: when a new LLVM is released, the homebrew installation directory changes, so the builds will fail. - # We either need to upgrade LLVM or change the directory being pointed to. - - name: Native macOS (release) - if: runner.os == 'macOS' + - name: Install dependencies run: | brew update - find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete - brew install llvm@${{ matrix.llvm }} + brew install llvm@${{ env.LLVM_VERSION }} + - name: Build + run: | export SDKROOT="$(xcrun --show-sdk-path)" - export PATH="/usr/local/opt/llvm@${{ matrix.llvm }}/bin:$PATH" - export PATH="/opt/homebrew/opt/llvm@${{ matrix.llvm }}/bin:$PATH" - CC=clang-20 ./configure --with-tail-call-interp + export PATH="/usr/local/opt/llvm@${{ env.LLVM_VERSION }}/bin:$PATH" + export PATH="/opt/homebrew/opt/llvm@${{ env.LLVM_VERSION }}/bin:$PATH" + CC=clang-${{ env.LLVM_VERSION }} ./configure --with-tail-call-interp make all --jobs 4 + - name: Test + run: | ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - name: Native Linux (debug) - if: runner.os == 'Linux' && matrix.target != 'free-threading' + linux: + name: ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-unknown-linux-gnu/gcc + runner: ubuntu-24.04 + configure_flags: --with-pydebug + - target: x86_64-unknown-linux-gnu/gcc-free-threading + runner: ubuntu-24.04 + configure_flags: --disable-gil + - target: aarch64-unknown-linux-gnu/gcc + runner: ubuntu-24.04-arm + configure_flags: --with-pydebug + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - name: Build run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - CC=clang-20 ./configure --with-tail-call-interp --with-pydebug + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ env.LLVM_VERSION }} + export PATH="$(llvm-config-${{ env.LLVM_VERSION }} --bindir):$PATH" + CC=clang-${{ env.LLVM_VERSION }} ./configure --with-tail-call-interp ${{ matrix.configure_flags }} make all --jobs 4 - ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - - name: Native Linux with free-threading (release) - if: matrix.target == 'free-threading' + - name: Test run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - CC=clang-20 ./configure --with-tail-call-interp --disable-gil - make all --jobs 4 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 463e7bf3355..cb40f6abc0b 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -25,10 +25,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3' - name: Compare checksum of bundled wheels to the ones published on PyPI diff --git a/.github/workflows/verify-expat.yml b/.github/workflows/verify-expat.yml new file mode 100644 index 00000000000..472a11db2da --- /dev/null +++ b/.github/workflows/verify-expat.yml @@ -0,0 +1,32 @@ +name: Verify bundled libexpat + +on: + workflow_dispatch: + push: + paths: + - 'Modules/expat/**' + - '.github/workflows/verify-expat.yml' + pull_request: + paths: + - 'Modules/expat/**' + - '.github/workflows/verify-expat.yml' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Download and verify bundled libexpat files + run: | + ./Modules/expat/refresh.sh + git diff --exit-code Modules/expat/ diff --git a/.github/zizmor.yml b/.github/zizmor.yml index 9b42b47cc85..7c776d5ea1f 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -1,10 +1,6 @@ -# Configuration for the zizmor static analysis tool, run via pre-commit in CI -# https://woodruffw.github.io/zizmor/configuration/ +# Configuration for the zizmor static analysis tool, run via prek in CI +# https://docs.zizmor.sh/configuration/ rules: dangerous-triggers: ignore: - documentation-links.yml - unpinned-uses: - config: - policies: - "*": ref-pin diff --git a/.gitignore b/.gitignore index e842676d866..3bf5187d531 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ *.cover *.iml *.o +*.o.tmp *.lto *.a *.so @@ -45,6 +46,7 @@ gmon.out .pytest_cache/ .ruff_cache/ .DS_Store +.pixi/ *.exe @@ -71,15 +73,15 @@ Lib/test/data/* /Makefile /Makefile.pre /iOSTestbed.* -iOS/Frameworks/ -iOS/Resources/Info.plist -iOS/testbed/build -iOS/testbed/Python.xcframework/ios-*/bin -iOS/testbed/Python.xcframework/ios-*/include -iOS/testbed/Python.xcframework/ios-*/lib -iOS/testbed/Python.xcframework/ios-*/Python.framework -iOS/testbed/iOSTestbed.xcodeproj/project.xcworkspace -iOS/testbed/iOSTestbed.xcodeproj/xcuserdata +Apple/iOS/Frameworks/ +Apple/iOS/Resources/Info.plist +Apple/testbed/build +Apple/testbed/Python.xcframework/*/bin +Apple/testbed/Python.xcframework/*/include +Apple/testbed/Python.xcframework/*/lib +Apple/testbed/Python.xcframework/*/Python.framework +Apple/testbed/*Testbed.xcodeproj/project.xcworkspace +Apple/testbed/*Testbed.xcodeproj/xcuserdata Mac/Makefile Mac/PythonLauncher/Info.plist Mac/PythonLauncher/Makefile @@ -135,9 +137,8 @@ Tools/unicode/data/ /config.log /config.status /config.status.lineno -# hendrikmuhs/ccache-action@v1 /.ccache -/cross-build/ +/cross-build*/ /jit_stencils*.h /platform /profile-clean-stamp diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2e0e67e7284..6878a7d92e3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,56 +1,79 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.8 + rev: e05c5c0818279e5ac248ac9e954431ba58865e61 # frozen: v0.15.7 hooks: - - id: ruff + - id: ruff-check + name: Run Ruff (lint) on Platforms/Apple/ + args: [--exit-non-zero-on-fix, --config=Platforms/Apple/.ruff.toml] + files: ^Platforms/Apple/ + - id: ruff-check name: Run Ruff (lint) on Doc/ args: [--exit-non-zero-on-fix] files: ^Doc/ - - id: ruff + - id: ruff-check name: Run Ruff (lint) on Lib/test/ args: [--exit-non-zero-on-fix] files: ^Lib/test/ - - id: ruff + - id: ruff-check + name: Run Ruff (lint) on Platforms/WASI/ + args: [--exit-non-zero-on-fix, --config=Platforms/WASI/.ruff.toml] + files: ^Platforms/WASI/ + - id: ruff-check name: Run Ruff (lint) on Tools/build/ args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] files: ^Tools/build/ - - id: ruff + - id: ruff-check name: Run Ruff (lint) on Tools/i18n/ args: [--exit-non-zero-on-fix, --config=Tools/i18n/.ruff.toml] files: ^Tools/i18n/ - - id: ruff + - id: ruff-check name: Run Ruff (lint) on Argument Clinic args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml] files: ^Tools/clinic/|Lib/test/test_clinic.py - - id: ruff + - id: ruff-check name: Run Ruff (lint) on Tools/peg_generator/ args: [--exit-non-zero-on-fix, --config=Tools/peg_generator/.ruff.toml] files: ^Tools/peg_generator/ + - id: ruff-check + name: Run Ruff (lint) on Tools/wasm/ + args: [--exit-non-zero-on-fix, --config=Tools/wasm/.ruff.toml] + files: ^Tools/wasm/ + - id: ruff-format + name: Run Ruff (format) on Platforms/Apple/ + args: [--exit-non-zero-on-fix, --config=Platforms/Apple/.ruff.toml] + files: ^Platforms/Apple/ - id: ruff-format name: Run Ruff (format) on Doc/ - args: [--check] + args: [--exit-non-zero-on-fix] files: ^Doc/ + - id: ruff-format + name: Run Ruff (format) on Platforms/WASI/ + args: [--exit-non-zero-on-fix, --config=Platforms/WASI/.ruff.toml] + files: ^Platforms/WASI/ - id: ruff-format name: Run Ruff (format) on Tools/build/check_warnings.py - args: [--check, --config=Tools/build/.ruff.toml] + args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] files: ^Tools/build/check_warnings.py + - id: ruff-format + name: Run Ruff (format) on Tools/wasm/ + args: [--exit-non-zero-on-fix, --config=Tools/wasm/.ruff.toml] + files: ^Tools/wasm/ - repo: https://github.com/psf/black-pre-commit-mirror - rev: 25.1.0 + rev: ea488cebbfd88a5f50b8bd95d5c829d0bb76feb8 # frozen: 26.1.0 hooks: - id: black name: Run Black on Tools/jit/ files: ^Tools/jit/ - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.5.5 + rev: ad1b27d73581aa16cca06fc4a0761fc563ffe8e8 # frozen: v1.5.6 hooks: - id: remove-tabs types: [python] - exclude: ^Tools/c-analyzer/cpython/_parser.py - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v6.0.0 + rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0 hooks: - id: check-case-conflict - id: check-merge-conflict @@ -62,30 +85,33 @@ repos: exclude: Lib/test/tokenizedata/coding20731.py - id: end-of-file-fixer files: '^\.github/CODEOWNERS$' + - id: mixed-line-ending + args: [--fix=auto] + exclude: '^Lib/test/.*data/' - id: trailing-whitespace types_or: [c, inc, python, rst, yaml] - id: trailing-whitespace files: '^\.github/CODEOWNERS|\.(gram)$' - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.33.2 + rev: 9f48a48aa91a6040d749ad68ec70907d907a5a7f # frozen: 0.37.0 hooks: - id: check-dependabot - id: check-github-workflows - id: check-readthedocs - repo: https://github.com/rhysd/actionlint - rev: v1.7.7 + rev: 393031adb9afb225ee52ae2ccd7a5af5525e03e8 # frozen: v1.7.11 hooks: - id: actionlint - - repo: https://github.com/woodruffw/zizmor-pre-commit - rev: v1.11.0 + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: b546b77c44c466a54a42af5499dcc0dcc1a3193f # frozen: v1.22.0 hooks: - id: zizmor - repo: https://github.com/sphinx-contrib/sphinx-lint - rev: v1.0.0 + rev: c883505f64b59c3c5c9375191e4ad9f98e727ccd # frozen: v1.0.2 hooks: - id: sphinx-lint args: [--enable=default-role] diff --git a/Android/testbed/app/src/main/python/android_testbed_main.py b/Android/testbed/app/src/main/python/android_testbed_main.py deleted file mode 100644 index 31b8e5343a8..00000000000 --- a/Android/testbed/app/src/main/python/android_testbed_main.py +++ /dev/null @@ -1,48 +0,0 @@ -import os -import runpy -import shlex -import signal -import sys - -# Some tests use SIGUSR1, but that's blocked by default in an Android app in -# order to make it available to `sigwait` in the Signal Catcher thread. -# (https://cs.android.com/android/platform/superproject/+/android14-qpr3-release:art/runtime/signal_catcher.cc). -# That thread's functionality is only useful for debugging the JVM, so disabling -# it should not weaken the tests. -# -# There's no safe way of stopping the thread completely (#123982), but simply -# unblocking SIGUSR1 is enough to fix most tests. -# -# However, in tests that generate multiple different signals in quick -# succession, it's possible for SIGUSR1 to arrive while the main thread is busy -# running the C-level handler for a different signal. In that case, the SIGUSR1 -# may be sent to the Signal Catcher thread instead, which will generate a log -# message containing the text "reacting to signal". -# -# Such tests may need to be changed in one of the following ways: -# * Use a signal other than SIGUSR1 (e.g. test_stress_delivery_simultaneous in -# test_signal.py). -# * Send the signal to a specific thread rather than the whole process (e.g. -# test_signals in test_threadsignals.py. -signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1]) - -mode = os.environ["PYTHON_MODE"] -module = os.environ["PYTHON_MODULE"] -sys.argv[1:] = shlex.split(os.environ["PYTHON_ARGS"]) - -cwd = f"{sys.prefix}/cwd" -if not os.path.exists(cwd): - # Empty directories are lost in the asset packing/unpacking process. - os.mkdir(cwd) -os.chdir(cwd) - -if mode == "-c": - # In -c mode, sys.path starts with an empty string, which means whatever the current - # working directory is at the moment of each import. - sys.path.insert(0, "") - exec(module, {}) -elif mode == "-m": - sys.path.insert(0, os.getcwd()) - runpy.run_module(module, run_name="__main__", alter_sys=True) -else: - raise ValueError(f"unknown mode: {mode}") diff --git a/Doc/.ruff.toml b/Doc/.ruff.toml index 3e676e13c3f..6b573fd58d0 100644 --- a/Doc/.ruff.toml +++ b/Doc/.ruff.toml @@ -32,6 +32,9 @@ ignore = [ "E501", # Ignore line length errors (we use auto-formatting) ] +[lint.per-file-ignores] +"tools/check-html-ids.py" = ["I001"] # Unsorted imports + [format] preview = true quote-style = "preserve" diff --git a/Doc/Makefile b/Doc/Makefile index 84578c5c57f..60970d50833 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -13,7 +13,7 @@ JOBS = auto PAPER = SOURCES = DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py) -REQUIREMENTS = requirements.txt +REQUIREMENTS = pylock.toml SPHINXERRORHANDLING = --fail-on-warning # Internal variables. @@ -58,7 +58,7 @@ build: @if [ -f ../Misc/NEWS ] ; then \ echo "Using existing Misc/NEWS file"; \ cp ../Misc/NEWS build/NEWS; \ - elif $(BLURB) help >/dev/null 2>&1 && $(SPHINXBUILD) --version >/dev/null 2>&1; then \ + elif $(BLURB) --version && $(SPHINXBUILD) --version ; then \ if [ -d ../Misc/NEWS.d ]; then \ echo "Building NEWS from Misc/NEWS.d with blurb"; \ $(BLURB) merge -f build/NEWS; \ @@ -88,6 +88,7 @@ htmlhelp: build "build/htmlhelp/pydoc.hhp project file." .PHONY: latex +latex: _ensure-sphinxcontrib-svg2pdfconverter latex: BUILDER = latex latex: build @echo "Build finished; the LaTeX files are in build/latex." @@ -140,7 +141,8 @@ doctest: pydoc-topics: BUILDER = pydoc-topics pydoc-topics: build @echo "Building finished; now run this:" \ - "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" + "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" \ + "&& cp build/pydoc-topics/module_docs.py ../Lib/pydoc_data/module_docs.py" .PHONY: gettext gettext: BUILDER = gettext @@ -184,7 +186,7 @@ venv: fi .PHONY: dist-no-html -dist-no-html: dist-text dist-pdf dist-epub dist-texinfo +dist-no-html: dist-text dist-epub dist-texinfo .PHONY: dist dist: @@ -230,7 +232,7 @@ dist-text: @echo "Build finished and archived!" .PHONY: dist-pdf -dist-pdf: +dist-pdf: _ensure-sphinxcontrib-svg2pdfconverter # archive the A4 latex @echo "Building LaTeX (A4 paper)..." mkdir -p dist @@ -241,7 +243,8 @@ dist-pdf: # as otherwise the full latexmk process is run twice. # ($$ is needed to escape the $; https://www.gnu.org/software/make/manual/make.html#Basics-of-Variable-References) -sed -i 's/: all-$$(FMT)/:/' build/latex/Makefile - (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`nproc`+1)) --output-sync LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) + if [ -n "$(filter output-sync,$(value .FEATURES))" ]; then OUTPUTSYNC=--output-sync; else OUTPUTSYNC=; fi && \ + (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`getconf _NPROCESSORS_ONLN`+1)) $$OUTPUTSYNC LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-a4.zip cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2 @echo "Build finished and archived!" @@ -290,6 +293,10 @@ _ensure-pre-commit: _ensure-sphinx-autobuild: $(MAKE) _ensure-package PACKAGE=sphinx-autobuild +.PHONY: _ensure-sphinxcontrib-svg2pdfconverter +_ensure-sphinxcontrib-svg2pdfconverter: + $(MAKE) _ensure-package PACKAGE=sphinxcontrib-svg2pdfconverter + .PHONY: check check: _ensure-pre-commit $(VENVDIR)/bin/python3 -m pre_commit run --all-files @@ -334,3 +341,9 @@ autobuild-stable-html: exit 1;; \ esac @$(MAKE) autobuild-dev-html + +# Collect HTML IDs to a JSON document +.PHONY: html-ids +html-ids: + $(PYTHON) tools/check-html-ids.py collect build/html \ + -o build/html/html-ids.json.gz diff --git a/Doc/_static/profiling-sampling-visualization.css b/Doc/_static/profiling-sampling-visualization.css new file mode 100644 index 00000000000..6bfbec3b8a6 --- /dev/null +++ b/Doc/_static/profiling-sampling-visualization.css @@ -0,0 +1,570 @@ +/** + * Sampling Profiler Visualization - Scoped CSS + */ + +.sampling-profiler-viz { + /* Match docs background colors */ + --bg-page: #ffffff; + --bg-panel: #ffffff; + --bg-subtle: #f8f8f8; + --bg-code: #f8f8f8; + + /* Match docs border style */ + --border-color: #e1e4e8; + --border-accent: #3776ab; + + /* Match docs text colors */ + --text-primary: #0d0d0d; + --text-secondary: #505050; + --text-muted: #6e6e6e; + --text-code: #333333; + + /* Accent colors */ + --color-python-blue: #306998; + --color-green: #388e3c; + --color-orange: #e65100; + --color-purple: #7b1fa2; + --color-red: #c62828; + --color-teal: #00897b; + --color-yellow: #d4a910; + --color-highlight: #fff9e6; + + --radius-lg: 8px; + --radius-md: 6px; + --radius-sm: 4px; + + /* Lighter shadows to match docs style */ + --shadow-card: 0 1px 3px rgba(0, 0, 0, 0.08); + --shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.04); + + --container-height: 520px; + --code-panel-width: 320px; + + /* Reset for isolation */ + font-family: var(--font-ui); + line-height: 1.5; + font-weight: 400; + color: var(--text-primary); + background-color: var(--bg-page); + font-synthesis: none; + text-rendering: optimizeLegibility; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + + /* Layout */ + position: relative; + width: 100%; + max-width: 920px; + height: var(--container-height); + display: grid; + grid-template-columns: var(--code-panel-width) 1fr; + margin: 24px auto; + border-radius: var(--radius-lg); + overflow: hidden; + box-shadow: var(--shadow-card); + border: 1px solid var(--border-color); + background: var(--bg-panel); + /* Prevent any DOM changes inside from affecting page scroll */ + contain: strict; +} + +.sampling-profiler-viz * { + box-sizing: border-box; +} + +/* Code Panel - Left Column */ +.sampling-profiler-viz #code-panel { + background: var(--bg-panel); + border-right: 1px solid var(--border-color); + overflow-y: auto; + font-family: var(--font-mono); + font-size: 12px; + line-height: 1.6; + display: flex; + flex-direction: column; +} + +.sampling-profiler-viz #code-panel .code-panel-title { + padding: 12px 16px; + font-size: 10px; + font-weight: 600; + color: var(--text-muted); + border-bottom: 1px solid var(--border-color); + background: var(--bg-code); + text-transform: uppercase; + letter-spacing: 1px; + flex-shrink: 0; +} + +.sampling-profiler-viz #code-panel .code-container { + margin: 0; + padding: 12px 0; + overflow-x: auto; + flex: 1; +} + +.sampling-profiler-viz #code-panel .line { + display: flex; + padding: 1px 0; + min-height: 20px; + transition: background-color 0.1s ease; +} + +.sampling-profiler-viz #code-panel .line-number { + color: var(--text-muted); + min-width: 40px; + text-align: right; + padding-right: 12px; + padding-left: 12px; + user-select: none; + flex-shrink: 0; + font-size: 11px; +} + +.sampling-profiler-viz #code-panel .line-content { + flex: 1; + color: var(--text-code); + padding-right: 12px; + white-space: pre; +} + +.sampling-profiler-viz #code-panel .line.highlighted { + background: var(--color-highlight); + border-left: 3px solid var(--color-yellow); +} + +.sampling-profiler-viz #code-panel .line.highlighted .line-number { + color: var(--color-yellow); + padding-left: 9px; +} + +.sampling-profiler-viz #code-panel .line.highlighted .line-content { + font-weight: 600; +} + +/* Python Syntax Highlighting */ +.sampling-profiler-viz #code-panel .keyword { + color: var(--color-red); + font-weight: 600; +} + +.sampling-profiler-viz #code-panel .function { + color: var(--color-purple); + font-weight: 600; +} + +.sampling-profiler-viz #code-panel .number { + color: var(--color-python-blue); +} + +.sampling-profiler-viz #code-panel .string { + color: #032f62; +} + +.sampling-profiler-viz #code-panel .comment { + color: #6a737d; + font-style: italic; +} + +.sampling-profiler-viz #code-panel .builtin { + color: var(--color-python-blue); +} + +/* Visualization Column - Right Side */ +.sampling-profiler-viz .viz-column { + display: flex; + flex-direction: column; + background: var(--bg-subtle); + overflow: hidden; +} + +/* Stack Section */ +.sampling-profiler-viz .stack-section { + padding: 12px 16px; + flex: 1; + min-height: 150px; + overflow-y: auto; +} + +.sampling-profiler-viz .stack-section-title { + font-size: 10px; + font-weight: 600; + color: var(--text-muted); + text-transform: uppercase; + letter-spacing: 1px; + margin-bottom: 10px; +} + +.sampling-profiler-viz .stack-visualization { + display: flex; + flex-direction: column; + gap: 4px; + min-height: 80px; +} + +/* Stack Frames - Vertical Layout */ +.sampling-profiler-viz .stack-frame { + position: relative; + width: 100%; + height: 32px; + cursor: pointer; + contain: layout style paint; + opacity: 0; + transform: translateY(-10px); +} + +.sampling-profiler-viz .stack-frame.visible { + opacity: 1; + transform: translateY(0); + transition: + opacity 0.3s ease, + transform 0.3s ease; +} + +.sampling-profiler-viz .stack-frame-bg { + position: absolute; + inset: 0; + border-radius: var(--radius-sm); + transition: opacity 0.15s; +} + +.sampling-profiler-viz .stack-frame-text { + position: absolute; + left: 10px; + top: 50%; + transform: translateY(-50%); + font: 500 11px var(--font-mono); + color: white; + pointer-events: none; +} + +.sampling-profiler-viz .stack-frame-flash { + position: absolute; + inset: 0; + background: white; + border-radius: var(--radius-sm); + opacity: 0; + pointer-events: none; +} + +.sampling-profiler-viz .stack-frame:hover .stack-frame-bg { + opacity: 0.85; +} + +/* Flying frames */ +.sampling-profiler-viz .stack-frame.flying { + pointer-events: none; + z-index: 1000; + position: fixed; + top: 0; + left: 0; + width: auto; + height: 32px; + opacity: 1; +} + +/* Flying stack clone */ +.stack-visualization.flying-clone { + transform-origin: center center; + will-change: transform, opacity; +} + +/* Sampling Panel */ +.sampling-profiler-viz .sampling-panel { + margin: 0 16px 12px 16px; + background: var(--bg-panel); + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + box-shadow: var(--shadow-sm); + overflow: hidden; + display: flex; + flex-direction: column; + flex: 0 0 auto; + height: 200px; + /* Lock font size to prevent Sphinx responsive scaling */ + font-size: 12px; +} + +.sampling-profiler-viz .sampling-header { + padding: 8px 10px; + border-bottom: 1px solid var(--border-color); + flex-shrink: 0; +} + +.sampling-profiler-viz .sampling-title { + font: 600 10px var(--font-mono); + color: var(--text-primary); + margin: 0 0 3px 0; +} + +.sampling-profiler-viz .sampling-stats { + font: 400 9px var(--font-mono); + color: var(--text-secondary); + display: flex; + gap: 12px; +} + +.sampling-profiler-viz .sampling-stats .missed { + color: var(--color-red); +} + +.sampling-profiler-viz .sampling-bars { + flex: 1; + padding: 10px 12px; + overflow-y: auto; +} + +.sampling-profiler-viz .sampling-bar-row { + display: flex; + align-items: center; + height: 22px; + gap: 8px; +} + +.sampling-profiler-viz .bar-label { + font: 500 8px var(--font-mono); + color: var(--text-primary); + flex-shrink: 0; + width: 60px; + overflow: hidden; + text-overflow: ellipsis; +} + +.sampling-profiler-viz .bar-container { + flex: 1; + height: 12px; + background: var(--border-color); + border-radius: 3px; + position: relative; + overflow: hidden; +} + +.sampling-profiler-viz .bar-fill { + height: 100%; + border-radius: 3px; + transition: width 0.2s ease; + min-width: 2px; +} + +.sampling-profiler-viz .bar-percent { + font: 500 8px var(--font-mono); + color: var(--text-secondary); + width: 36px; + text-align: right; + flex-shrink: 0; +} + +/* Impact effect circle */ +.impact-circle { + position: fixed; + width: 30px; + height: 30px; + border-radius: 50%; + background: var(--color-teal); + transform: translate(-50%, -50%); + pointer-events: none; + z-index: 2000; +} + +/* Control Panel - Integrated */ +.sampling-profiler-viz #control-panel { + display: flex; + align-items: center; + gap: 12px; + padding: 12px 16px; + background: var(--bg-panel); + border-top: 1px solid var(--border-color); + flex-shrink: 0; + flex-wrap: wrap; +} + +.sampling-profiler-viz .control-group { + display: flex; + gap: 6px; + align-items: center; +} + +.sampling-profiler-viz .control-group label { + font-size: 10px; + color: var(--text-muted); + font-weight: 500; + white-space: nowrap; +} + +.sampling-profiler-viz .control-btn { + background: var(--bg-panel); + color: var(--text-primary); + border: 1px solid var(--border-color); + padding: 6px 10px; + border-radius: var(--radius-sm); + cursor: pointer; + transition: all 0.15s ease; + font-family: var(--font-mono); + font-size: 11px; + font-weight: 500; + display: flex; + align-items: center; + gap: 4px; +} + +.sampling-profiler-viz .control-btn:hover { + background: var(--bg-subtle); + border-color: var(--text-muted); +} + +.sampling-profiler-viz .control-btn:active { + transform: scale(0.98); +} + +.sampling-profiler-viz .control-btn.active { + background: var(--color-python-blue); + color: white; + border-color: var(--color-python-blue); +} + +.sampling-profiler-viz .control-btn.active:hover { + background: #2f6493; +} + +/* Timeline Scrubber */ +.sampling-profiler-viz .timeline-scrubber { + flex: 1; + display: flex; + align-items: center; + gap: 8px; + min-width: 160px; +} + +.sampling-profiler-viz #timeline-scrubber { + flex: 1; + height: 5px; + border-radius: 3px; + background: var(--border-color); + outline: none; + appearance: none; + -webkit-appearance: none; + cursor: pointer; + min-width: 60px; +} + +.sampling-profiler-viz #timeline-scrubber::-webkit-slider-thumb { + -webkit-appearance: none; + width: 14px; + height: 14px; + border-radius: 50%; + background: var(--color-python-blue); + cursor: pointer; + transition: transform 0.15s; + box-shadow: 0 2px 6px rgba(0, 0, 0, 0.2); +} + +.sampling-profiler-viz #timeline-scrubber::-webkit-slider-thumb:hover { + transform: scale(1.15); +} + +.sampling-profiler-viz #timeline-scrubber::-moz-range-thumb { + width: 14px; + height: 14px; + border-radius: 50%; + background: var(--color-python-blue); + cursor: pointer; + border: none; + box-shadow: 0 2px 6px rgba(0, 0, 0, 0.2); +} + +.sampling-profiler-viz #time-display { + font: 500 10px var(--font-mono); + color: var(--text-secondary); + min-width: 90px; + text-align: right; + font-variant-numeric: tabular-nums; +} + +/* Sample Interval Slider */ +.sampling-profiler-viz #sample-interval { + width: 80px; + height: 4px; + border-radius: 2px; + background: var(--border-color); + outline: none; + appearance: none; + -webkit-appearance: none; + cursor: pointer; + flex-shrink: 0; +} + +.sampling-profiler-viz #sample-interval::-webkit-slider-thumb { + -webkit-appearance: none; + width: 12px; + height: 12px; + border-radius: 50%; + background: var(--color-teal); + cursor: pointer; + transition: transform 0.15s; + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2); +} + +.sampling-profiler-viz #sample-interval::-webkit-slider-thumb:hover { + transform: scale(1.15); +} + +.sampling-profiler-viz #sample-interval::-moz-range-thumb { + width: 12px; + height: 12px; + border-radius: 50%; + background: var(--color-teal); + cursor: pointer; + border: none; +} + +.sampling-profiler-viz #interval-display { + font: 500 9px var(--font-mono); + color: var(--text-secondary); + min-width: 40px; + font-variant-numeric: tabular-nums; +} + +/* Flash overlay */ +.sampling-profiler-viz .flash-overlay { + position: absolute; + inset: 0; + background: white; + pointer-events: none; + opacity: 0; +} + +/* Performance optimizations */ +.sampling-profiler-viz .stack-frame, +.sampling-profiler-viz .flying-frame, +.sampling-profiler-viz .sampling-bar-row { + will-change: transform, opacity; + contain: layout style paint; +} + +/* Reduced motion support */ +@media (prefers-reduced-motion: reduce) { + .sampling-profiler-viz .stack-frame, + .sampling-profiler-viz .flying-frame, + .sampling-profiler-viz .sampling-bar-row, + .impact-circle { + animation-duration: 0.01ms !important; + transition-duration: 0.01ms !important; + } +} + +/* Responsive adjustments for narrower viewports */ +@media (max-width: 800px) { + .sampling-profiler-viz { + grid-template-columns: 280px 1fr; + --container-height: 550px; + } + + .sampling-profiler-viz #code-panel { + font-size: 11px; + } + + .sampling-profiler-viz .control-btn { + padding: 5px 8px; + font-size: 10px; + } +} diff --git a/Doc/_static/profiling-sampling-visualization.js b/Doc/_static/profiling-sampling-visualization.js new file mode 100644 index 00000000000..3729be6795c --- /dev/null +++ b/Doc/_static/profiling-sampling-visualization.js @@ -0,0 +1,1163 @@ +/** + * Sampling Profiler Visualization + */ +(function () { + "use strict"; + + // ============================================================================ + // Configuration + // ============================================================================ + + const TIMINGS = { + sampleIntervalMin: 100, + sampleIntervalMax: 500, + sampleIntervalDefault: 200, + sampleToFlame: 600, + defaultSpeed: 0.05, + }; + + const LAYOUT = { frameSpacing: 6 }; + + // Function name to color mapping + const FUNCTION_COLORS = { + main: "#306998", + fibonacci: "#D4A910", + add: "#E65100", + multiply: "#7B1FA2", + calculate: "#D4A910", + }; + const DEFAULT_FUNCTION_COLOR = "#306998"; + + // Easing functions - cubic-bezier approximations + const EASING_MAP = { + linear: "linear", + easeOutQuad: "cubic-bezier(0.25, 0.46, 0.45, 0.94)", + easeOutCubic: "cubic-bezier(0.215, 0.61, 0.355, 1)", + }; + + function getFunctionColor(funcName) { + return FUNCTION_COLORS[funcName] || DEFAULT_FUNCTION_COLOR; + } + + // ============================================================================ + // Animation Manager + // ============================================================================ + + class AnimationManager { + constructor() { + this.activeAnimations = new Set(); + } + + to(element, props, duration, easing = "easeOutQuad", onComplete = null) { + this.killAnimationsOf(element); + + const cssEasing = EASING_MAP[easing] || EASING_MAP.easeOutQuad; + + const transformProps = {}; + const otherProps = {}; + + for (const [key, value] of Object.entries(props)) { + if (key === "position") { + if (typeof value.x === "number") transformProps.x = value.x; + if (typeof value.y === "number") transformProps.y = value.y; + } else if (key === "x" || key === "y") { + transformProps[key] = value; + } else if (key === "scale") { + transformProps.scale = value; + } else if (key === "alpha" || key === "opacity") { + otherProps.opacity = value; + } else { + otherProps[key] = value; + } + } + + const computedStyle = getComputedStyle(element); + const matrix = new DOMMatrix(computedStyle.transform); + const currentScale = Math.sqrt( + matrix.m11 * matrix.m11 + matrix.m21 * matrix.m21, + ); + + transformProps.x ??= matrix.m41; + transformProps.y ??= matrix.m42; + transformProps.scale ??= currentScale; + + const initialTransform = this._buildTransformString( + matrix.m41, + matrix.m42, + currentScale, + ); + + const finalTransform = this._buildTransformString( + transformProps.x, + transformProps.y, + transformProps.scale, + ); + + const initialKeyframe = { transform: initialTransform }; + const finalKeyframe = { transform: finalTransform }; + + for (const [key, value] of Object.entries(otherProps)) { + const currentVal = + key === "opacity" + ? element.style.opacity || computedStyle.opacity + : element.style[key]; + initialKeyframe[key] = currentVal; + finalKeyframe[key] = value; + } + + const animation = element.animate([initialKeyframe, finalKeyframe], { + duration, + easing: cssEasing, + fill: "forwards", + }); + + this.activeAnimations.add(animation); + animation.onfinish = () => { + this.activeAnimations.delete(animation); + element.style.transform = finalTransform; + for (const [key, value] of Object.entries(finalKeyframe)) { + if (key !== "transform") { + element.style[key] = typeof value === "number" ? `${value}` : value; + } + } + if (onComplete) onComplete(); + }; + + return animation; + } + + killAnimationsOf(element) { + element.getAnimations().forEach((animation) => animation.cancel()); + this.activeAnimations.forEach((animation) => { + if (animation.effect && animation.effect.target === element) { + animation.cancel(); + this.activeAnimations.delete(animation); + } + }); + } + + _buildTransformString(x, y, scale = 1) { + return `translate(${x}px, ${y}px) scale(${scale})`; + } + } + + const anim = new AnimationManager(); + + // ============================================================================ + // Execution Trace Model + // ============================================================================ + + class ExecutionEvent { + constructor( + type, + functionName, + lineno, + timestamp, + args = null, + value = null, + ) { + this.type = type; + this.functionName = functionName; + this.lineno = lineno; + this.timestamp = timestamp; + this.args = args; + this.value = value; + } + } + + class ExecutionTrace { + constructor(source, events) { + this.source = source; + this.events = events.map( + (e) => + new ExecutionEvent(e.type, e.func, e.line, e.ts, e.args, e.value), + ); + this.duration = events.length > 0 ? events[events.length - 1].ts : 0; + } + + getEventsUntil(timestamp) { + return this.events.filter((e) => e.timestamp <= timestamp); + } + + getStackAt(timestamp) { + const stack = []; + const events = this.getEventsUntil(timestamp); + + for (const event of events) { + if (event.type === "call") { + stack.push({ + func: event.functionName, + line: event.lineno, + args: event.args, + }); + } else if (event.type === "return") { + stack.pop(); + } else if (event.type === "line") { + if (stack.length > 0) { + stack[stack.length - 1].line = event.lineno; + } + } + } + return stack; + } + + getNextEvent(timestamp) { + return this.events.find((e) => e.timestamp > timestamp); + } + + getSourceLines() { + return this.source.split("\n"); + } + } + + // ============================================================================ + // Demo Data + // ============================================================================ + + // This placeholder is replaced by the profiling_trace Sphinx extension + // during the documentation build with dynamically generated trace data. + const DEMO_SIMPLE = /* PROFILING_TRACE_DATA */ null; + + // ============================================================================ + // Code Panel Component + // ============================================================================ + + class CodePanel { + constructor(source) { + this.source = source; + this.currentLine = null; + + this.element = document.createElement("div"); + this.element.id = "code-panel"; + + const title = document.createElement("div"); + title.className = "code-panel-title"; + title.textContent = "source code"; + this.element.appendChild(title); + + this.codeContainer = document.createElement("pre"); + this.codeContainer.className = "code-container"; + this.element.appendChild(this.codeContainer); + + this._renderSource(); + } + + updateSource(source) { + this.source = source; + this.codeContainer.innerHTML = ""; + this._renderSource(); + this.currentLine = null; + } + + _renderSource() { + const lines = this.source.split("\n"); + + lines.forEach((line, index) => { + const lineNumber = index + 1; + const lineDiv = document.createElement("div"); + lineDiv.className = "line"; + lineDiv.dataset.line = lineNumber; + + const lineNumSpan = document.createElement("span"); + lineNumSpan.className = "line-number"; + lineNumSpan.textContent = lineNumber; + lineDiv.appendChild(lineNumSpan); + + const codeSpan = document.createElement("span"); + codeSpan.className = "line-content"; + codeSpan.innerHTML = this._highlightSyntax(line); + lineDiv.appendChild(codeSpan); + + this.codeContainer.appendChild(lineDiv); + }); + } + + _highlightSyntax(line) { + return line + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/(f?"[^"]*"|f?'[^']*')/g, '$1') + .replace(/(#.*$)/g, '$1') + .replace( + /\b(def|if|elif|else|return|for|in|range|print|__name__|__main__)\b/g, + '$1', + ) + .replace( + /def<\/span>\s+(\w+)/g, + 'def $1', + ) + .replace(/\b(\d+)\b/g, '$1'); + } + + highlightLine(lineNumber) { + if (this.currentLine === lineNumber) return; + + if (this.currentLine !== null) { + const prevLine = this.codeContainer.querySelector( + `[data-line="${this.currentLine}"]`, + ); + if (prevLine) prevLine.classList.remove("highlighted"); + } + + if (lineNumber === null || lineNumber === undefined) { + this.currentLine = null; + return; + } + + this.currentLine = lineNumber; + const newLine = this.codeContainer.querySelector( + `[data-line="${lineNumber}"]`, + ); + if (newLine) { + newLine.classList.add("highlighted"); + } + } + + reset() { + this.highlightLine(null); + this.codeContainer.scrollTop = 0; + } + + destroy() { + this.element.remove(); + } + } + + // ============================================================================ + // Stack Frame Component + // ============================================================================ + + class DOMStackFrame { + constructor(functionName, lineno, args = null) { + this.functionName = functionName; + this.lineno = lineno; + this.args = args; + this.isActive = false; + this.color = getFunctionColor(functionName); + + this.element = document.createElement("div"); + this.element.className = "stack-frame"; + this.element.dataset.function = functionName; + + this.bgElement = document.createElement("div"); + this.bgElement.className = "stack-frame-bg"; + this.bgElement.style.backgroundColor = this.color; + this.element.appendChild(this.bgElement); + + this.textElement = document.createElement("span"); + this.textElement.className = "stack-frame-text"; + this.textElement.textContent = functionName; + this.element.appendChild(this.textElement); + + this.flashElement = document.createElement("div"); + this.flashElement.className = "stack-frame-flash"; + this.element.appendChild(this.flashElement); + + this.element.addEventListener("pointerover", this._onHover.bind(this)); + this.element.addEventListener("pointerout", this._onHoverOut.bind(this)); + } + + destroy() { + this.element.parentNode?.removeChild(this.element); + } + + updateLine(lineno) { + this.lineno = lineno; + this.textElement.textContent = this.functionName; + } + + setActive(isActive) { + if (this.isActive === isActive) return; + this.isActive = isActive; + this.bgElement.style.opacity = isActive ? "1.0" : "0.9"; + } + + _onHover() { + this.bgElement.style.opacity = "0.8"; + } + + _onHoverOut() { + this.bgElement.style.opacity = this.isActive ? "1.0" : "0.9"; + } + + flash(duration = 150) { + this.flashElement.animate([{ opacity: 1 }, { opacity: 0 }], { + duration, + easing: "ease-out", + }); + } + + getPosition() { + const rect = this.element.getBoundingClientRect(); + return { x: rect.left, y: rect.top }; + } + } + + // ============================================================================ + // Stack Visualization Component + // ============================================================================ + + class DOMStackVisualization { + constructor() { + this.frames = []; + this.frameSpacing = LAYOUT.frameSpacing; + + this.element = document.createElement("div"); + this.element.className = "stack-visualization"; + } + + processEvent(event) { + if (event.type === "call") { + this.pushFrame(event.functionName, event.lineno, event.args); + } else if (event.type === "return") { + this.popFrame(); + } else if (event.type === "line") { + this.updateTopFrameLine(event.lineno); + } + } + + updateTopFrameLine(lineno) { + if (this.frames.length > 0) { + this.frames[this.frames.length - 1].updateLine(lineno); + } + } + + pushFrame(functionName, lineno, args = null) { + if (this.frames.length > 0) { + this.frames[this.frames.length - 1].setActive(false); + } + + const frame = new DOMStackFrame(functionName, lineno, args); + frame.setActive(true); + this.element.appendChild(frame.element); + this.frames.push(frame); + + requestAnimationFrame(() => { + frame.element.classList.add("visible"); + }); + } + + popFrame() { + if (this.frames.length === 0) return; + + const frame = this.frames.pop(); + frame.element.classList.remove("visible"); + setTimeout(() => frame.destroy(), 300); + + if (this.frames.length > 0) { + this.frames[this.frames.length - 1].setActive(true); + } + } + + clear() { + this.frames.forEach((frame) => frame.destroy()); + this.frames = []; + this.element.innerHTML = ""; + } + + flashAll() { + this.frames.forEach((frame) => frame.flash()); + } + + createStackClone(container) { + const clone = this.element.cloneNode(false); + clone.className = "stack-visualization flying-clone"; + + const elementRect = this.element.getBoundingClientRect(); + const containerRect = container.getBoundingClientRect(); + + // Position relative to container since contain: strict makes position:fixed relative to container + clone.style.position = "absolute"; + clone.style.left = elementRect.left - containerRect.left + "px"; + clone.style.top = elementRect.top - containerRect.top + "px"; + clone.style.width = elementRect.width + "px"; + clone.style.pointerEvents = "none"; + clone.style.zIndex = "1000"; + + this.frames.forEach((frame) => { + const frameClone = frame.element.cloneNode(true); + frameClone.classList.add("visible"); + frameClone.style.opacity = "1"; + frameClone.style.transform = "translateY(0)"; + frameClone.style.transition = "none"; + clone.appendChild(frameClone); + }); + + container.appendChild(clone); + return clone; + } + + updateToMatch(targetStack) { + while (this.frames.length > targetStack.length) { + this.popFrame(); + } + + targetStack.forEach(({ func, line, args }, index) => { + if (index < this.frames.length) { + const frame = this.frames[index]; + if (frame.functionName !== func) { + frame.updateLine(line); + } + frame.setActive(index === targetStack.length - 1); + } else { + this.pushFrame(func, line, args); + } + }); + + if (this.frames.length > 0) { + this.frames[this.frames.length - 1].setActive(true); + } + } + } + + // ============================================================================ + // Sampling Panel Component + // ============================================================================ + + class DOMSamplingPanel { + constructor() { + this.samples = []; + this.functionCounts = {}; + this.totalSamples = 0; + this.sampleInterval = TIMINGS.sampleIntervalDefault; + this.groundTruthFunctions = new Set(); + this.bars = {}; + + this.element = document.createElement("div"); + this.element.className = "sampling-panel"; + + const header = document.createElement("div"); + header.className = "sampling-header"; + + const title = document.createElement("h3"); + title.className = "sampling-title"; + title.textContent = "Sampling Profiler"; + header.appendChild(title); + + const stats = document.createElement("div"); + stats.className = "sampling-stats"; + + this.sampleCountEl = document.createElement("span"); + this.sampleCountEl.textContent = "Samples: 0"; + stats.appendChild(this.sampleCountEl); + + this.intervalEl = document.createElement("span"); + this.intervalEl.textContent = `Interval: ${this.sampleInterval}ms`; + stats.appendChild(this.intervalEl); + + this.missedFunctionsEl = document.createElement("span"); + this.missedFunctionsEl.className = "missed"; + stats.appendChild(this.missedFunctionsEl); + + header.appendChild(stats); + this.element.appendChild(header); + + this.barsContainer = document.createElement("div"); + this.barsContainer.className = "sampling-bars"; + this.element.appendChild(this.barsContainer); + } + + setSampleInterval(interval) { + this.sampleInterval = interval; + this.intervalEl.textContent = `Interval: ${interval}ms`; + } + + setGroundTruth(allFunctions) { + this.groundTruthFunctions = new Set(allFunctions); + this._updateMissedCount(); + } + + addSample(stack) { + this.totalSamples++; + this.sampleCountEl.textContent = `Samples: ${this.totalSamples}`; + + stack.forEach((frame) => { + const funcName = frame.func; + this.functionCounts[funcName] = + (this.functionCounts[funcName] || 0) + 1; + }); + + this._updateBars(); + this._updateMissedCount(); + } + + reset() { + this.samples = []; + this.functionCounts = {}; + this.totalSamples = 0; + this.sampleCountEl.textContent = "Samples: 0"; + this.missedFunctionsEl.textContent = ""; + this.barsContainer.innerHTML = ""; + this.bars = {}; + } + + _updateMissedCount() { + if (this.groundTruthFunctions.size === 0) return; + + const capturedFunctions = new Set(Object.keys(this.functionCounts)); + const notYetSeen = [...this.groundTruthFunctions].filter( + (f) => !capturedFunctions.has(f), + ); + + if (notYetSeen.length > 0) { + this.missedFunctionsEl.textContent = `Not yet seen: ${notYetSeen.length}`; + this.missedFunctionsEl.classList.add("missed"); + this.missedFunctionsEl.style.color = ""; + } else if (this.totalSamples > 0) { + this.missedFunctionsEl.textContent = "All captured!"; + this.missedFunctionsEl.classList.remove("missed"); + this.missedFunctionsEl.style.color = "var(--color-green)"; + } else { + this.missedFunctionsEl.textContent = ""; + } + } + + _updateBars() { + const sorted = Object.entries(this.functionCounts).sort( + (a, b) => b[1] - a[1], + ); + + sorted.forEach(([funcName, count], index) => { + const percentage = + this.totalSamples > 0 ? count / this.totalSamples : 0; + + if (!this.bars[funcName]) { + const row = this._createBarRow(funcName); + this.barsContainer.appendChild(row); + this.bars[funcName] = row; + } + + const row = this.bars[funcName]; + const barFill = row.querySelector(".bar-fill"); + barFill.style.width = `${percentage * 100}%`; + + const percentEl = row.querySelector(".bar-percent"); + percentEl.textContent = `${(percentage * 100).toFixed(0)}%`; + + const currentIndex = Array.from(this.barsContainer.children).indexOf( + row, + ); + if (currentIndex !== index) { + this.barsContainer.insertBefore( + row, + this.barsContainer.children[index], + ); + } + }); + } + + _createBarRow(funcName) { + const row = document.createElement("div"); + row.className = "sampling-bar-row"; + row.dataset.function = funcName; + + const label = document.createElement("span"); + label.className = "bar-label"; + label.textContent = funcName; + row.appendChild(label); + + const barContainer = document.createElement("div"); + barContainer.className = "bar-container"; + + const barFill = document.createElement("div"); + barFill.className = "bar-fill"; + barFill.style.backgroundColor = getFunctionColor(funcName); + barContainer.appendChild(barFill); + + row.appendChild(barContainer); + + const percent = document.createElement("span"); + percent.className = "bar-percent"; + percent.textContent = "0%"; + row.appendChild(percent); + + return row; + } + + getTargetPosition() { + const rect = this.barsContainer.getBoundingClientRect(); + return { x: rect.left + rect.width / 2, y: rect.top + 50 }; + } + + showImpactEffect(position) { + const impact = document.createElement("div"); + impact.className = "impact-circle"; + impact.style.position = "fixed"; + impact.style.left = `${position.x}px`; + impact.style.top = `${position.y}px`; + + // Append to barsContainer parent to avoid triggering scroll + this.element.appendChild(impact); + + impact.animate( + [ + { transform: "translate(-50%, -50%) scale(1)", opacity: 0.6 }, + { transform: "translate(-50%, -50%) scale(4)", opacity: 0 }, + ], + { + duration: 300, + easing: "ease-out", + }, + ).onfinish = () => impact.remove(); + } + } + + // ============================================================================ + // Control Panel Component + // ============================================================================ + + class ControlPanel { + constructor( + container, + onPlay, + onPause, + onReset, + onSpeedChange, + onSeek, + onStep, + onSampleIntervalChange = null, + ) { + this.container = container; + this.onPlay = onPlay; + this.onPause = onPause; + this.onReset = onReset; + this.onSpeedChange = onSpeedChange; + this.onSeek = onSeek; + this.onStep = onStep; + this.onSampleIntervalChange = onSampleIntervalChange; + + this.isPlaying = false; + this.speed = TIMINGS.defaultSpeed; + + this._createControls(); + } + + _createControls() { + const panel = document.createElement("div"); + panel.id = "control-panel"; + + const sampleIntervalHtml = this.onSampleIntervalChange + ? ` +
+ + + ${TIMINGS.sampleIntervalDefault}ms +
+ ` + : ""; + + panel.innerHTML = ` +
+ + + +
+ + ${sampleIntervalHtml} + +
+ + 0ms +
+ `; + + this.container.appendChild(panel); + + this.playPauseBtn = panel.querySelector("#play-pause-btn"); + this.resetBtn = panel.querySelector("#reset-btn"); + this.stepBtn = panel.querySelector("#step-btn"); + this.scrubber = panel.querySelector("#timeline-scrubber"); + this.timeDisplay = panel.querySelector("#time-display"); + + this.playPauseBtn.addEventListener("click", () => + this._togglePlayPause(), + ); + this.resetBtn.addEventListener("click", () => this._handleReset()); + this.stepBtn.addEventListener("click", () => this._handleStep()); + this.scrubber.addEventListener("input", (e) => this._handleSeek(e)); + + if (this.onSampleIntervalChange) { + this.sampleIntervalSlider = panel.querySelector("#sample-interval"); + this.intervalDisplay = panel.querySelector("#interval-display"); + this.sampleIntervalSlider.addEventListener("input", (e) => + this._handleSampleIntervalChange(e), + ); + } + } + + _handleSampleIntervalChange(e) { + const interval = parseInt(e.target.value); + this.intervalDisplay.textContent = `${interval}ms`; + this.onSampleIntervalChange(interval); + } + + _togglePlayPause() { + this.isPlaying = !this.isPlaying; + + if (this.isPlaying) { + this.playPauseBtn.textContent = "⏸ Pause"; + this.playPauseBtn.classList.add("active"); + this.onPlay(); + } else { + this.playPauseBtn.textContent = "▶ Play"; + this.playPauseBtn.classList.remove("active"); + this.onPause(); + } + } + + _handleReset() { + this.isPlaying = false; + this.playPauseBtn.textContent = "▶ Play"; + this.playPauseBtn.classList.remove("active"); + this.scrubber.value = 0; + this.timeDisplay.textContent = "0ms"; + this.onReset(); + } + + _handleStep() { + if (this.onStep) this.onStep(); + } + + _handleSeek(e) { + const percentage = parseFloat(e.target.value); + this.onSeek(percentage / 100); + } + + updateTimeDisplay(currentTime, totalTime) { + this.timeDisplay.textContent = `${Math.floor(currentTime)}ms / ${Math.floor(totalTime)}ms`; + const percentage = (currentTime / totalTime) * 100; + this.scrubber.value = percentage; + } + + setDuration(duration) { + this.duration = duration; + } + + pause() { + if (this.isPlaying) this._togglePlayPause(); + } + + destroy() { + const panel = this.container.querySelector("#control-panel"); + if (panel) panel.remove(); + } + } + + // ============================================================================ + // Visual Effects Manager + // ============================================================================ + + class VisualEffectsManager { + constructor(container) { + this.container = container; + this.flyingAnimationInProgress = false; + + this.flashOverlay = document.createElement("div"); + this.flashOverlay.className = "flash-overlay"; + this.container.appendChild(this.flashOverlay); + } + + triggerSamplingEffect(stackViz, samplingPanel, currentTime, trace) { + if (this.flyingAnimationInProgress) return; + + const stack = trace.getStackAt(currentTime); + + if (stack.length === 0) { + samplingPanel.addSample(stack); + return; + } + + this.flyingAnimationInProgress = true; + stackViz.flashAll(); + + const clone = stackViz.createStackClone(this.container); + const targetPosition = samplingPanel.getTargetPosition(); + + this._animateFlash(); + this._animateFlyingStack(clone, targetPosition, () => { + samplingPanel.showImpactEffect(targetPosition); + clone.remove(); + + const currentStack = trace.getStackAt(currentTime); + samplingPanel.addSample(currentStack); + this.flyingAnimationInProgress = false; + }); + } + + _animateFlash() { + anim.to(this.flashOverlay, { opacity: 0.1 }, 0).onfinish = () => { + anim.to(this.flashOverlay, { opacity: 0 }, 150, "easeOutQuad"); + }; + } + + _animateFlyingStack(clone, targetPosition, onComplete) { + const containerRect = this.container.getBoundingClientRect(); + const cloneRect = clone.getBoundingClientRect(); + + // Convert viewport coordinates to container-relative + const startX = cloneRect.left - containerRect.left + cloneRect.width / 2; + const startY = cloneRect.top - containerRect.top + cloneRect.height / 2; + const targetX = targetPosition.x - containerRect.left; + const targetY = targetPosition.y - containerRect.top; + + const deltaX = targetX - startX; + const deltaY = targetY - startY; + + anim.to( + clone, + { + x: deltaX, + y: deltaY, + scale: 0.3, + opacity: 0.6, + }, + TIMINGS.sampleToFlame, + "easeOutCubic", + onComplete, + ); + } + } + + // ============================================================================ + // Main Visualization Class + // ============================================================================ + + class SamplingVisualization { + constructor(container) { + this.container = container; + + this.trace = new ExecutionTrace(DEMO_SIMPLE.source, DEMO_SIMPLE.trace); + + this.currentTime = 0; + this.isPlaying = false; + this.playbackSpeed = TIMINGS.defaultSpeed; + this.eventIndex = 0; + + this.sampleInterval = TIMINGS.sampleIntervalDefault; + this.lastSampleTime = 0; + + this._createLayout(); + + this.effectsManager = new VisualEffectsManager(this.vizColumn); + + this.lastTime = performance.now(); + this._animate(); + } + + _createLayout() { + this.codePanel = new CodePanel(this.trace.source); + this.container.appendChild(this.codePanel.element); + + this.vizColumn = document.createElement("div"); + this.vizColumn.className = "viz-column"; + this.container.appendChild(this.vizColumn); + + const stackSection = document.createElement("div"); + stackSection.className = "stack-section"; + + const stackTitle = document.createElement("div"); + stackTitle.className = "stack-section-title"; + stackTitle.textContent = "Call Stack"; + stackSection.appendChild(stackTitle); + + this.stackViz = new DOMStackVisualization(); + stackSection.appendChild(this.stackViz.element); + this.vizColumn.appendChild(stackSection); + + this.samplingPanel = new DOMSamplingPanel(); + this.samplingPanel.setGroundTruth(this._getGroundTruthFunctions()); + this.vizColumn.appendChild(this.samplingPanel.element); + + this.controls = new ControlPanel( + this.vizColumn, + () => this.play(), + () => this.pause(), + () => this.reset(), + (speed) => this.setSpeed(speed), + (progress) => this.seek(progress), + () => this.step(), + (interval) => this.setSampleInterval(interval), + ); + this.controls.setDuration(this.trace.duration); + } + + _getGroundTruthFunctions() { + const functions = new Set(); + this.trace.events.forEach((event) => { + if (event.type === "call") { + functions.add(event.functionName); + } + }); + return [...functions]; + } + + play() { + this.isPlaying = true; + } + + pause() { + this.isPlaying = false; + } + + reset() { + this.currentTime = 0; + this.eventIndex = 0; + this.isPlaying = false; + this.lastSampleTime = 0; + this.stackViz.clear(); + this.codePanel.reset(); + this.samplingPanel.reset(); + this.controls.updateTimeDisplay(0, this.trace.duration); + } + + setSpeed(speed) { + this.playbackSpeed = speed; + } + + setSampleInterval(interval) { + this.sampleInterval = interval; + this.samplingPanel.setSampleInterval(interval); + } + + seek(progress) { + this.currentTime = progress * this.trace.duration; + this.eventIndex = 0; + this.lastSampleTime = 0; + this._rebuildState(); + } + + step() { + this.pause(); + + const nextEvent = this.trace.getNextEvent(this.currentTime); + + if (nextEvent) { + // Calculate delta to reach next event + epsilon + const targetTime = nextEvent.timestamp + 0.1; + const delta = targetTime - this.currentTime; + if (delta > 0) { + this._advanceTime(delta); + } + } + } + + _animate(currentTime = performance.now()) { + const deltaTime = currentTime - this.lastTime; + this.lastTime = currentTime; + + this.update(deltaTime); + requestAnimationFrame((t) => this._animate(t)); + } + + update(deltaTime) { + if (!this.isPlaying) { + this.controls.updateTimeDisplay(this.currentTime, this.trace.duration); + return; + } + + const virtualDelta = deltaTime * this.playbackSpeed; + this._advanceTime(virtualDelta); + } + + _advanceTime(virtualDelta) { + this.currentTime += virtualDelta; + + if (this.currentTime >= this.trace.duration) { + this.currentTime = this.trace.duration; + this.isPlaying = false; + this.controls.pause(); + } + + while (this.eventIndex < this.trace.events.length) { + const event = this.trace.events[this.eventIndex]; + + if (event.timestamp > this.currentTime) break; + + this._processEvent(event); + this.eventIndex++; + } + + this.controls.updateTimeDisplay(this.currentTime, this.trace.duration); + + if (this.currentTime - this.lastSampleTime >= this.sampleInterval) { + this._takeSample(); + this.lastSampleTime = this.currentTime; + } + } + + _processEvent(event) { + this.stackViz.processEvent(event); + + if (event.type === "call") { + this.codePanel.highlightLine(event.lineno); + } else if (event.type === "return") { + const currentStack = this.trace.getStackAt(this.currentTime); + if (currentStack.length > 0) { + this.codePanel.highlightLine( + currentStack[currentStack.length - 1].line, + ); + } else { + this.codePanel.highlightLine(null); + } + } else if (event.type === "line") { + this.codePanel.highlightLine(event.lineno); + } + } + + _takeSample() { + this.effectsManager.triggerSamplingEffect( + this.stackViz, + this.samplingPanel, + this.currentTime, + this.trace, + ); + } + + _rebuildState() { + this.stackViz.clear(); + this.codePanel.reset(); + this.samplingPanel.reset(); + + for (let t = 0; t < this.currentTime; t += this.sampleInterval) { + const stack = this.trace.getStackAt(t); + this.samplingPanel.addSample(stack); + this.lastSampleTime = t; + } + + const stack = this.trace.getStackAt(this.currentTime); + this.stackViz.updateToMatch(stack); + + if (stack.length > 0) { + this.codePanel.highlightLine(stack[stack.length - 1].line); + } + + this.eventIndex = this.trace.getEventsUntil(this.currentTime).length; + } + } + + // ============================================================================ + // Initialize + // ============================================================================ + + function init() { + // If trace data hasn't been injected yet (local dev), don't initialize + if (!DEMO_SIMPLE) return; + + const appContainer = document.getElementById("sampling-profiler-viz"); + if (appContainer) { + new SamplingVisualization(appContainer); + } + } + + if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", init); + } else { + init(); + } +})(); diff --git a/Doc/_static/tachyon-example-flamegraph.html b/Doc/_static/tachyon-example-flamegraph.html new file mode 100644 index 00000000000..701c959bda3 --- /dev/null +++ b/Doc/_static/tachyon-example-flamegraph.html @@ -0,0 +1,3260 @@ + + + + + + Tachyon Profiler - Flamegraph Report + + + + + + + +
+ +
+
+ + Tachyon + + Flamegraph Report +
+
+ + +
+
+ + + + + + + + +
+
+ + +
+ + + + +
+
+
+
+ + +
+ + Tachyon Profiler + + + Python Sampling Profiler + + + + + +
+
+ + + + diff --git a/Doc/_static/tachyon-example-heatmap.html b/Doc/_static/tachyon-example-heatmap.html new file mode 100644 index 00000000000..f725e947500 --- /dev/null +++ b/Doc/_static/tachyon-example-heatmap.html @@ -0,0 +1,3804 @@ + + + + + + /tmp/tachyon_selfcontained.py - Heatmap + + + +
+ +
+
+ + Tachyon + + /tmp/tachyon_selfcontained.py +
+
+ + + + + + + + + + + +
+
+ + +
+
+
+
1,054
+
Self Samples
+
+
+
4,236
+
Cumulative
+
+
+
24
+
Lines Hit
+
+
+
100.00%
+
% of Total
+
+
+
206
+
Max Self
+
+
+
1054
+
Max Total
+
+
+
+ + +
+
+ Intensity: +
+
+ Cold + + Hot +
+ +
+
+ Self Time +
+ Total Time +
+
+ Show All +
+ Hot Only +
+
+ Heat +
+ Specialization +
+ + +
+
+
+ + +
+
+
Line
+
Self
+
Total
+
Code
+
+
+
1
+
+
+
+
"""
+ +
+
+
2
+
+
+
+
Tachyon Demo - Self-contained profiling example.
+ +
+
+
3
+
+
+
+
Pure Python with no external imports for clean heatmap output.
+ +
+
+
4
+
+
+
+
"""
+ +
+
+
5
+
+
+
+
+ +
+
+
6
+
+
+
+
+ +
+
+
7
+
1
+
1
+ +
def fibonacci(n):
+ +
+ +
+
8
+
+
+
+
"""Recursive fibonacci - creates deep call stacks."""
+ +
+
+
9
+
3
+
3
+ +
if n <= 1:
+ +
+ +
+
10
+
17
+
17
+ +
return n
+
+
+ +
+
11
+
206
+
227
+ +
return fibonacci(n - 1) + fibonacci(n - 2)
+
+
+ +
+
12
+
+
+
+
+ +
+
+
13
+
+
+
+
+ +
+
+
14
+
+
+
+
def bubble_sort(arr):
+ +
+
+
15
+
+
+
+
"""Classic bubble sort - O(n^2) comparison sorting."""
+ +
+
+
16
+
+
+
+
n = len(arr)
+ +
+
+
17
+
+
+
+
for i in range(n):
+ +
+
+
18
+
2
+
2
+ +
for j in range(0, n - i - 1):
+ +
+ +
+
19
+
8
+
8
+ +
if arr[j] > arr[j + 1]:
+
+
+ +
+
20
+
2
+
2
+ +
arr[j], arr[j + 1] = arr[j + 1], arr[j]
+ +
+ +
+
21
+
+
+
+
return arr
+ +
+
+
22
+
+
+
+
+ +
+
+
23
+
+
+
+
+ +
+
+
24
+
+
+
+
def matrix_multiply(a, b):
+ +
+
+
25
+
+
+
+
"""Matrix multiplication using nested loops."""
+ +
+
+
26
+
+
+
+
rows_a, cols_a = len(a), len(a[0])
+ +
+
+
27
+
+
+
+
rows_b, cols_b = len(b), len(b[0])
+ +
+
+
28
+
+
+
+
result = [[0] * cols_b for _ in range(rows_a)]
+ +
+
+
29
+
+
+
+
+ +
+
+
30
+
+
+
+
for i in range(rows_a):
+ +
+
+
31
+
+
+
+
for j in range(cols_b):
+ +
+
+
32
+
8
+
8
+ +
for k in range(cols_a):
+ +
+ +
+
33
+
62
+
62
+ +
result[i][j] += a[i][k] * b[k][j]
+
+
+ +
+
34
+
+
+
+
return result
+ +
+
+
35
+
+
+
+
+ +
+
+
36
+
+
+
+
+ +
+
+
37
+
+
+
+
def prime_sieve(limit):
+ +
+
+
38
+
+
+
+
"""Sieve of Eratosthenes - find all primes up to limit."""
+ +
+
+
39
+
+
+
+
is_prime = [True] * (limit + 1)
+ +
+
+
40
+
+
+
+
is_prime[0] = is_prime[1] = False
+ +
+
+
41
+
+
+
+
+ +
+
+
42
+
+
+
+
for num in range(2, int(limit ** 0.5) + 1):
+ +
+
+
43
+
+
+
+
if is_prime[num]:
+ +
+
+
44
+
1
+
1
+ +
for multiple in range(num * num, limit + 1, num):
+
+
+ +
+
45
+
+
+
+
is_prime[multiple] = False
+ +
+
+
46
+
+
+
+
+ +
+
+
47
+
2
+
2
+ +
return [num for num, prime in enumerate(is_prime) if prime]
+ +
+ +
+
48
+
+
+
+
+ +
+
+
49
+
+
+
+
+ +
+
+
50
+
+
+
+
def string_processing(iterations):
+ +
+
+
51
+
+
+
+
"""String operations - concatenation and formatting."""
+ +
+
+
52
+
+
+
+
for _ in range(iterations):
+ +
+
+
53
+
+
+
+
result = ""
+ +
+
+
54
+
+
+
+
for i in range(50):
+ +
+
+
55
+
91
+
91
+ +
result += f"item_{i}_"
+
+
+ +
+
56
+
22
+
22
+ +
parts = result.split("_")
+ +
+ +
+
57
+
10
+
10
+ +
joined = "-".join(parts)
+ +
+ +
+
58
+
+
+
+
+ +
+
+
59
+
+
+
+
+ +
+
+
60
+
+
+
+
def list_operations(iterations):
+ +
+
+
61
+
+
+
+
"""List comprehensions and operations."""
+ +
+
+
62
+
+
+
+
for _ in range(iterations):
+ +
+
+
63
+
118
+
118
+ +
squares = [x ** 2 for x in range(500)]
+ +
+ +
+
64
+
119
+
119
+ +
evens = [x for x in squares if x % 2 == 0]
+
+
+ +
+
65
+
12
+
12
+ +
total = sum(evens)
+ +
+ +
+
66
+
+
+
+
+ +
+
+
67
+
+
+
+
+ +
+
+
68
+
+
+
+
def dict_operations(iterations):
+ +
+
+
69
+
+
+
+
"""Dictionary creation and lookups."""
+ +
+
+
70
+
+
+
+
for _ in range(iterations):
+ +
+
+
71
+
206
+
206
+ +
data = {f"key_{i}": i ** 2 for i in range(200)}
+
+
+ +
+
72
+
158
+
158
+ +
values = [data[f"key_{i}"] for i in range(200)]
+ +
+ +
+
73
+
5
+
5
+ +
total = sum(values)
+ +
+ +
+
74
+
+
+
+
+ +
+
+
75
+
+
+
+
+ +
+
+
76
+
+
+
+
def compute_heavy():
+ +
+
+
77
+
+
+
+
"""CPU-intensive computation section."""
+ +
+
+
78
+
1
+
301
+ +
fibonacci(30)
+
+
+ +
+
79
+
+
+
+
+ +
+
+
80
+
+
+
+
size = 60
+ +
+
+
81
+
+
+
+
a = [[i + j for j in range(size)] for i in range(size)]
+ +
+
+
82
+
+
+
+
b = [[i * j for j in range(size)] for i in range(size)]
+ +
+
+
83
+
+
+
+
matrix_multiply(a, b)
+ +
+
+
84
+
+
+
+
+ +
+
+
85
+
+
+
+
prime_sieve(10000)
+ +
+
+
86
+
+
+
+
+ +
+
+
87
+
+
+
+
+ +
+
+
88
+
+
+
+
def data_processing():
+ +
+
+
89
+
+
+
+
"""Data structure operations."""
+ +
+
+
90
+
+
753
+ +
string_processing(2000)
+
+
+ +
+
91
+
+
+
+
list_operations(1500)
+ +
+
+
92
+
+
+
+
dict_operations(1000)
+ +
+
+
93
+
+
+
+
+ +
+
+
94
+
+
+
+
data = list(range(800))
+ +
+
+
95
+
+
+
+
for _ in range(3):
+ +
+
+
96
+
+
+
+
data = data[::-1]
+ +
+
+
97
+
+
+
+
bubble_sort(data[:200])
+ +
+
+
98
+
+
+
+
+ +
+
+
99
+
+
+
+
+ +
+
+
100
+
+
+
+
def main():
+ +
+
+
101
+
+
+
+
"""Main entry point."""
+ +
+
+
102
+
+
1,054
+ +
compute_heavy()
+
+
+ +
+
103
+
+
+
+
data_processing()
+ +
+
+
104
+
+
+
+
+ +
+
+
105
+
+
+
+
+ +
+
+
106
+
+
+
+
if __name__ == "__main__":
+ +
+
+
107
+
+
1,054
+ +
main()
+
+
+ + +
+
+ + + + diff --git a/Doc/about.rst b/Doc/about.rst index 8f635d7f743..5c1b497ca6b 100644 --- a/Doc/about.rst +++ b/Doc/about.rst @@ -32,8 +32,9 @@ Contributors to the Python documentation ---------------------------------------- Many people have contributed to the Python language, the Python standard -library, and the Python documentation. See :source:`Misc/ACKS` in the Python -source distribution for a partial list of contributors. +library, and the Python documentation. See the `CPython +GitHub repository `__ +for a partial list of contributors. It is only with the input and contributions of the Python community that Python has such wonderful documentation -- Thank You! diff --git a/Doc/bugs.rst b/Doc/bugs.rst index faf13eeb6a7..254a22f2622 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -9,7 +9,7 @@ stability. In order to maintain this reputation, the developers would like to know of any deficiencies you find in Python. It can be sometimes faster to fix bugs yourself and contribute patches to -Python as it streamlines the process and involves less people. Learn how to +Python as it streamlines the process and involves fewer people. Learn how to :ref:`contribute `. Documentation bugs @@ -19,6 +19,12 @@ If you find a bug in this documentation or would like to propose an improvement, please submit a bug report on the :ref:`issue tracker `. If you have a suggestion on how to fix it, include that as well. +.. only:: translation + + If the bug or suggested improvement concerns the translation of this + documentation, submit the report to the + `translation’s repository `_ instead. + You can also open a discussion item on our `Documentation Discourse forum `_. diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 59d913a0462..09c9ed3ca54 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -2,7 +2,7 @@ .. _allocating-objects: -Allocating Objects on the Heap +Allocating objects on the heap ============================== @@ -140,10 +140,6 @@ Allocating Objects on the Heap * :c:member:`~PyTypeObject.tp_alloc` -.. c:function:: void PyObject_Del(void *op) - - Same as :c:func:`PyObject_Free`. - .. c:var:: PyObject _Py_NoneStruct Object which is visible in Python as ``None``. This should only be accessed @@ -156,3 +152,37 @@ Allocating Objects on the Heap :ref:`moduleobjects` To allocate and create extension modules. + +Soft-deprecated aliases +^^^^^^^^^^^^^^^^^^^^^^^ + +.. soft-deprecated:: 3.15 + +These are aliases to existing functions and macros. +They exist solely for backwards compatibility. + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * Soft-deprecated alias + * Function + * * .. c:macro:: PyObject_NEW(type, typeobj) + * :c:macro:`PyObject_New` + * * .. c:macro:: PyObject_NEW_VAR(type, typeobj, n) + * :c:macro:`PyObject_NewVar` + * * .. c:macro:: PyObject_INIT(op, typeobj) + * :c:func:`PyObject_Init` + * * .. c:macro:: PyObject_INIT_VAR(op, typeobj, n) + * :c:func:`PyObject_InitVar` + * * .. c:macro:: PyObject_MALLOC(n) + * :c:func:`PyObject_Malloc` + * * .. c:macro:: PyObject_REALLOC(p, n) + * :c:func:`PyObject_Realloc` + * * .. c:macro:: PyObject_FREE(p) + * :c:func:`PyObject_Free` + * * .. c:macro:: PyObject_DEL(p) + * :c:func:`PyObject_Free` + * * .. c:macro:: PyObject_Del(p) + * :c:func:`PyObject_Free` diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst index 96050f59bd5..cb98d4307ee 100644 --- a/Doc/c-api/apiabiversion.rst +++ b/Doc/c-api/apiabiversion.rst @@ -34,6 +34,23 @@ See :ref:`stable` for a discussion of API and ABI stability across versions. This can be ``0xA`` for alpha, ``0xB`` for beta, ``0xC`` for release candidate or ``0xF`` for final. + + .. c:namespace:: NULL + .. c:macro:: PY_RELEASE_LEVEL_ALPHA + :no-typesetting: + .. c:macro:: PY_RELEASE_LEVEL_BETA + :no-typesetting: + .. c:macro:: PY_RELEASE_LEVEL_GAMMA + :no-typesetting: + .. c:macro:: PY_RELEASE_LEVEL_FINAL + :no-typesetting: + + For completeness, the values are available as macros: + :c:macro:`!PY_RELEASE_LEVEL_ALPHA` (``0xA``), + :c:macro:`!PY_RELEASE_LEVEL_BETA` (``0xB``), + :c:macro:`!PY_RELEASE_LEVEL_GAMMA` (``0xC``), and + :c:macro:`!PY_RELEASE_LEVEL_FINAL` (``0xF``). + .. c:macro:: PY_RELEASE_SERIAL The ``2`` in ``3.4.1a2``. Zero for final releases. @@ -46,6 +63,12 @@ See :ref:`stable` for a discussion of API and ABI stability across versions. Use this for numeric comparisons, for example, ``#if PY_VERSION_HEX >= ...``. +.. c:macro:: PY_VERSION + + The Python version as a string, for example, ``"3.4.1a2"``. + +These macros are defined in :source:`Include/patchlevel.h`. + Run-time version ---------------- diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 3429a4eb652..58456a36b96 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -160,7 +160,7 @@ There are three ways strings and buffers can be converted to C: ``w*`` (read-write :term:`bytes-like object`) [Py_buffer] This format accepts any object which implements the read-write buffer interface. It fills a :c:type:`Py_buffer` structure provided by the caller. - The buffer may contain embedded null bytes. The caller have to call + The buffer may contain embedded null bytes. The caller has to call :c:func:`PyBuffer_Release` when it is done with the buffer. ``es`` (:class:`str`) [const char \*encoding, char \*\*buffer] @@ -305,7 +305,7 @@ the minimal value for the corresponding signed integer type of the same size. ``D`` (:class:`complex`) [Py_complex] Convert a Python complex number to a C :c:type:`Py_complex` structure. -.. deprecated:: next +.. deprecated:: 3.15 For unsigned integer formats ``B``, ``H``, ``I``, ``k`` and ``K``, :exc:`DeprecationWarning` is emitted when the value is larger than @@ -516,6 +516,28 @@ API Functions } +.. c:function:: int PyArg_ParseArray(PyObject *const *args, Py_ssize_t nargs, const char *format, ...) + + Parse the parameters of a function that takes only array parameters into + local variables (that is, a function using the :c:macro:`METH_FASTCALL` + calling convention). + Returns true on success; on failure, it returns false and raises the + appropriate exception. + + .. versionadded:: 3.15 + + +.. c:function:: int PyArg_ParseArrayAndKeywords(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames, const char *format, const char * const *kwlist, ...) + + Parse the parameters of a function that takes both array and keyword + parameters into local variables (that is, a function using the + :c:macro:`METH_FASTCALL` ``|`` :c:macro:`METH_KEYWORDS` calling convention). + Returns true on success; on failure, it returns false and raises the + appropriate exception. + + .. versionadded:: 3.15 + + .. c:function:: int PyArg_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t max, ...) A simpler form of parameter retrieval which does not use a format string to diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index d3081894ead..dc3e0f37c36 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -10,11 +10,6 @@ Buffer Protocol --------------- -.. sectionauthor:: Greg Stein -.. sectionauthor:: Benjamin Peterson -.. sectionauthor:: Stefan Krah - - Certain objects available in Python wrap access to an underlying memory array or *buffer*. Such objects include the built-in :class:`bytes` and :class:`bytearray`, and some extension types like :class:`array.array`. @@ -261,6 +256,12 @@ readonly, format MUST be consistent for all consumers. For example, :c:expr:`PyBUF_SIMPLE | PyBUF_WRITABLE` can be used to request a simple writable buffer. + .. c:macro:: PyBUF_WRITEABLE + + This is an alias to :c:macro:`PyBUF_WRITABLE`. + + .. soft-deprecated:: 3.13 + .. c:macro:: PyBUF_FORMAT Controls the :c:member:`~Py_buffer.format` field. If set, this field MUST @@ -501,10 +502,11 @@ Buffer-related functions *indices* must point to an array of ``view->ndim`` indices. -.. c:function:: int PyBuffer_FromContiguous(const Py_buffer *view, const void *buf, Py_ssize_t len, char fort) +.. c:function:: int PyBuffer_FromContiguous(const Py_buffer *view, const void *buf, Py_ssize_t len, char order) Copy contiguous *len* bytes from *buf* to *view*. - *fort* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering). + *order* can be ``'C'`` or ``'F'`` or ``'A'`` (for C-style or Fortran-style + ordering or either one). ``0`` is returned on success, ``-1`` on error. diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index e2b22ec3c79..2b36da997d4 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -44,6 +44,10 @@ Direct API functions On failure, return ``NULL`` with an exception set. + .. note:: + If the object implements the buffer protocol, then the buffer + must not be mutated while the bytearray object is being created. + .. c:function:: PyObject* PyByteArray_FromStringAndSize(const char *string, Py_ssize_t len) @@ -58,6 +62,10 @@ Direct API functions On failure, return ``NULL`` with an exception set. + .. note:: + If the object implements the buffer protocol, then the buffer + must not be mutated while the bytearray object is being created. + .. c:function:: Py_ssize_t PyByteArray_Size(PyObject *bytearray) @@ -70,6 +78,9 @@ Direct API functions ``NULL`` pointer. The returned array always has an extra null byte appended. + .. note:: + It is not thread-safe to mutate the bytearray object while using the returned char array. + .. c:function:: int PyByteArray_Resize(PyObject *bytearray, Py_ssize_t len) @@ -89,6 +100,9 @@ These macros trade safety for speed and they don't check pointers. Similar to :c:func:`PyByteArray_AsString`, but without error checking. + .. note:: + It is not thread-safe to mutate the bytearray object while using the returned char array. + .. c:function:: Py_ssize_t PyByteArray_GET_SIZE(PyObject *bytearray) diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index d47beee68ea..f56bcd6333a 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -47,6 +47,10 @@ called with a non-bytes parameter. *len* on success, and ``NULL`` on failure. If *v* is ``NULL``, the contents of the bytes object are uninitialized. + .. soft-deprecated:: 3.15 + Use the :c:type:`PyBytesWriter` API instead of + ``PyBytes_FromStringAndSize(NULL, len)``. + .. c:function:: PyObject* PyBytes_FromFormat(const char *format, ...) @@ -123,6 +127,10 @@ called with a non-bytes parameter. Return the bytes representation of object *o* that implements the buffer protocol. + .. note:: + If the object implements the buffer protocol, then the buffer + must not be mutated while the bytes object is being created. + .. c:function:: Py_ssize_t PyBytes_Size(PyObject *o) @@ -181,6 +189,9 @@ called with a non-bytes parameter. created, the old reference to *bytes* will still be discarded and the value of *\*bytes* will be set to ``NULL``; the appropriate exception will be set. + .. note:: + If *newpart* implements the buffer protocol, then the buffer + must not be mutated while the new bytes object is being created. .. c:function:: void PyBytes_ConcatAndDel(PyObject **bytes, PyObject *newpart) @@ -188,6 +199,10 @@ called with a non-bytes parameter. appended to *bytes*. This version releases the :term:`strong reference` to *newpart* (i.e. decrements its reference count). + .. note:: + If *newpart* implements the buffer protocol, then the buffer + must not be mutated while the new bytes object is being created. + .. c:function:: PyObject* PyBytes_Join(PyObject *sep, PyObject *iterable) @@ -206,6 +221,9 @@ called with a non-bytes parameter. .. versionadded:: 3.14 + .. note:: + If *iterable* objects implement the buffer protocol, then the buffers + must not be mutated while the new bytes object is being created. .. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize) @@ -219,3 +237,212 @@ called with a non-bytes parameter. reallocation fails, the original bytes object at *\*bytes* is deallocated, *\*bytes* is set to ``NULL``, :exc:`MemoryError` is set, and ``-1`` is returned. + + .. soft-deprecated:: 3.15 + Use the :c:type:`PyBytesWriter` API instead. + + +.. c:function:: PyObject *PyBytes_Repr(PyObject *bytes, int smartquotes) + + Get the string representation of *bytes*. This function is currently used to + implement :meth:`!bytes.__repr__` in Python. + + This function does not do type checking; it is undefined behavior to pass + *bytes* as a non-bytes object or ``NULL``. + + If *smartquotes* is true, the representation will use a double-quoted string + instead of single-quoted string when single-quotes are present in *bytes*. + For example, the byte string ``'Python'`` would be represented as + ``b"'Python'"`` when *smartquotes* is true, or ``b'\'Python\''`` when it is + false. + + On success, this function returns a :term:`strong reference` to a + :class:`str` object containing the representation. On failure, this + returns ``NULL`` with an exception set. + + +.. c:function:: PyObject *PyBytes_DecodeEscape(const char *s, Py_ssize_t len, const char *errors, Py_ssize_t unicode, const char *recode_encoding) + + Unescape a backslash-escaped string *s*. *s* must not be ``NULL``. + *len* must be the size of *s*. + + *errors* must be one of ``"strict"``, ``"replace"``, or ``"ignore"``. If + *errors* is ``NULL``, then ``"strict"`` is used by default. + + On success, this function returns a :term:`strong reference` to a Python + :class:`bytes` object containing the unescaped string. On failure, this + function returns ``NULL`` with an exception set. + + .. versionchanged:: 3.9 + *unicode* and *recode_encoding* are now unused. + + +.. _pybyteswriter: + +PyBytesWriter +------------- + +The :c:type:`PyBytesWriter` API can be used to create a Python :class:`bytes` +object. + +.. versionadded:: 3.15 + +.. c:type:: PyBytesWriter + + A bytes writer instance. + + The API is **not thread safe**: a writer should only be used by a single + thread at the same time. + + The instance must be destroyed by :c:func:`PyBytesWriter_Finish` on + success, or :c:func:`PyBytesWriter_Discard` on error. + + +Create, Finish, Discard +^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: PyBytesWriter* PyBytesWriter_Create(Py_ssize_t size) + + Create a :c:type:`PyBytesWriter` to write *size* bytes. + + If *size* is greater than zero, allocate *size* bytes, and set the + writer size to *size*. The caller is responsible to write *size* + bytes using :c:func:`PyBytesWriter_GetData`. + This function does not overallocate. + + On error, set an exception and return ``NULL``. + + *size* must be positive or zero. + +.. c:function:: PyObject* PyBytesWriter_Finish(PyBytesWriter *writer) + + Finish a :c:type:`PyBytesWriter` created by + :c:func:`PyBytesWriter_Create`. + + On success, return a Python :class:`bytes` object. + On error, set an exception and return ``NULL``. + + The writer instance is invalid after the call in any case. + No API can be called on the writer after :c:func:`PyBytesWriter_Finish`. + +.. c:function:: PyObject* PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) + + Similar to :c:func:`PyBytesWriter_Finish`, but resize the writer + to *size* bytes before creating the :class:`bytes` object. + +.. c:function:: PyObject* PyBytesWriter_FinishWithPointer(PyBytesWriter *writer, void *buf) + + Similar to :c:func:`PyBytesWriter_Finish`, but resize the writer + using *buf* pointer before creating the :class:`bytes` object. + + Set an exception and return ``NULL`` if *buf* pointer is outside the + internal buffer bounds. + + Function pseudo-code:: + + Py_ssize_t size = (char*)buf - (char*)PyBytesWriter_GetData(writer); + return PyBytesWriter_FinishWithSize(writer, size); + +.. c:function:: void PyBytesWriter_Discard(PyBytesWriter *writer) + + Discard a :c:type:`PyBytesWriter` created by :c:func:`PyBytesWriter_Create`. + + Do nothing if *writer* is ``NULL``. + + The writer instance is invalid after the call. + No API can be called on the writer after :c:func:`PyBytesWriter_Discard`. + +High-level API +^^^^^^^^^^^^^^ + +.. c:function:: int PyBytesWriter_WriteBytes(PyBytesWriter *writer, const void *bytes, Py_ssize_t size) + + Grow the *writer* internal buffer by *size* bytes, + write *size* bytes of *bytes* at the *writer* end, + and add *size* to the *writer* size. + + If *size* is equal to ``-1``, call ``strlen(bytes)`` to get the + string length. + + On success, return ``0``. + On error, set an exception and return ``-1``. + +.. c:function:: int PyBytesWriter_Format(PyBytesWriter *writer, const char *format, ...) + + Similar to :c:func:`PyBytes_FromFormat`, but write the output directly at + the writer end. Grow the writer internal buffer on demand. Then add the + written size to the writer size. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + +Getters +^^^^^^^ + +.. c:function:: Py_ssize_t PyBytesWriter_GetSize(PyBytesWriter *writer) + + Get the writer size. + + The function cannot fail. + +.. c:function:: void* PyBytesWriter_GetData(PyBytesWriter *writer) + + Get the writer data: start of the internal buffer. + + The pointer is valid until :c:func:`PyBytesWriter_Finish` or + :c:func:`PyBytesWriter_Discard` is called on *writer*. + + The function cannot fail. + + +Low-level API +^^^^^^^^^^^^^ + +.. c:function:: int PyBytesWriter_Resize(PyBytesWriter *writer, Py_ssize_t size) + + Resize the writer to *size* bytes. It can be used to enlarge or to + shrink the writer. + This function typically overallocates to achieve amortized performance when + resizing multiple times. + + Newly allocated bytes are left uninitialized. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + *size* must be positive or zero. + +.. c:function:: int PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t grow) + + Resize the writer by adding *grow* bytes to the current writer size. + This function typically overallocates to achieve amortized performance when + resizing multiple times. + + Newly allocated bytes are left uninitialized. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + *size* can be negative to shrink the writer. + +.. c:function:: void* PyBytesWriter_GrowAndUpdatePointer(PyBytesWriter *writer, Py_ssize_t size, void *buf) + + Similar to :c:func:`PyBytesWriter_Grow`, but update also the *buf* + pointer. + + The *buf* pointer is moved if the internal buffer is moved in memory. + The *buf* relative position within the internal buffer is left + unchanged. + + On error, set an exception and return ``NULL``. + + *buf* must not be ``NULL``. + + Function pseudo-code:: + + Py_ssize_t pos = (char*)buf - (char*)PyBytesWriter_GetData(writer); + if (PyBytesWriter_Grow(writer, size) < 0) { + return NULL; + } + return (char*)PyBytesWriter_GetData(writer) + pos; diff --git a/Doc/c-api/call.rst b/Doc/c-api/call.rst index 7198d6bc056..9838879a528 100644 --- a/Doc/c-api/call.rst +++ b/Doc/c-api/call.rst @@ -347,6 +347,8 @@ please see individual documentation for details. .. versionadded:: 3.9 +.. c:function:: PyObject* _PyObject_Vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) + :no-typesetting: .. c:function:: PyObject* PyObject_Vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) @@ -358,7 +360,12 @@ please see individual documentation for details. Return the result of the call on success, or raise an exception and return *NULL* on failure. - .. versionadded:: 3.9 + .. versionadded:: 3.8 as ``_PyObject_Vectorcall`` + + .. versionchanged:: 3.9 + + Renamed to the current name, without the leading underscore. + The old provisional name is :term:`soft deprecated`. .. c:function:: PyObject* PyObject_VectorcallDict(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwdict) diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst index 64dc4f5275b..03a848d68ed 100644 --- a/Doc/c-api/capsule.rst +++ b/Doc/c-api/capsule.rst @@ -15,13 +15,19 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:type:: PyCapsule This subtype of :c:type:`PyObject` represents an opaque value, useful for C - extension modules who need to pass an opaque value (as a :c:expr:`void*` + extension modules which need to pass an opaque value (as a :c:expr:`void*` pointer) through Python code to other C code. It is often used to make a C function pointer defined in one module available to other modules, so the regular import mechanism can be used to access C APIs defined in dynamically loaded modules. +.. c:var:: PyTypeObject PyCapsule_Type + + The type object corresponding to capsule objects. This is the same object + as :class:`types.CapsuleType` in the Python layer. + + .. c:type:: PyCapsule_Destructor The type of a destructor callback for a capsule. Defined as:: diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index 61eb994c370..2501ed9580d 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -7,7 +7,7 @@ Cell Objects "Cell" objects are used to implement variables referenced by multiple scopes. For each such variable, a cell object is created to store the value; the local -variables of each stack frame that references the value contains a reference to +variables of each stack frame that references the value contain a reference to the cells from outer scopes which also use that variable. When the value is accessed, the value contained in the cell is used instead of the cell object itself. This de-referencing of the cell object requires support from the diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 717b0da8f87..57b77f92a7d 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -7,8 +7,6 @@ Code Objects ------------ -.. sectionauthor:: Jeffrey Yasskin - Code objects are a low-level detail of the CPython implementation. Each one represents a chunk of executable code that hasn't yet been bound into a function. @@ -69,13 +67,14 @@ bound into a function. The old name is deprecated, but will remain available until the signature changes again. +.. c:function:: PyCodeObject* PyCode_NewWithPosOnlyArgs(...) + :no-typesetting: + .. c:function:: PyCodeObject* PyUnstable_Code_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable) Similar to :c:func:`PyUnstable_Code_New`, but with an extra "posonlyargcount" for positional-only arguments. The same caveats that apply to ``PyUnstable_Code_New`` also apply to this function. - .. index:: single: PyCode_NewWithPosOnlyArgs (C function) - .. versionadded:: 3.8 as ``PyCode_NewWithPosOnlyArgs`` .. versionchanged:: 3.11 @@ -211,6 +210,19 @@ bound into a function. .. versionadded:: 3.12 +.. c:function:: PyObject *PyCode_Optimize(PyObject *code, PyObject *consts, PyObject *names, PyObject *lnotab_obj) + + This is a function that does nothing. + + Prior to Python 3.10, this function would perform basic optimizations to a + code object. + + .. versionchanged:: 3.10 + This function now does nothing. + + .. soft-deprecated:: 3.13 + + .. _c_codeobject_flags: Code Object Flags @@ -287,9 +299,12 @@ These functions are part of the unstable C API tier: this functionality is a CPython implementation detail, and the API may change without deprecation warnings. +.. c:function:: Py_ssize_t _PyEval_RequestCodeExtraIndex(freefunc free) + :no-typesetting: + .. c:function:: Py_ssize_t PyUnstable_Eval_RequestCodeExtraIndex(freefunc free) - Return a new an opaque index value used to adding data to code objects. + Return a new opaque index value used to adding data to code objects. You generally call this function once (per interpreter) and use the result with ``PyCode_GetExtra`` and ``PyCode_SetExtra`` to manipulate @@ -299,8 +314,6 @@ may change without deprecation warnings. *free* will be called on non-``NULL`` data stored under the new index. Use :c:func:`Py_DecRef` when storing :c:type:`PyObject`. - .. index:: single: _PyEval_RequestCodeExtraIndex (C function) - .. versionadded:: 3.6 as ``_PyEval_RequestCodeExtraIndex`` .. versionchanged:: 3.12 @@ -309,6 +322,9 @@ may change without deprecation warnings. The old private name is deprecated, but will be available until the API changes. +.. c:function:: int _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) + :no-typesetting: + .. c:function:: int PyUnstable_Code_GetExtra(PyObject *code, Py_ssize_t index, void **extra) Set *extra* to the extra data stored under the given index. @@ -317,8 +333,6 @@ may change without deprecation warnings. If no data was set under the index, set *extra* to ``NULL`` and return 0 without setting an exception. - .. index:: single: _PyCode_GetExtra (C function) - .. versionadded:: 3.6 as ``_PyCode_GetExtra`` .. versionchanged:: 3.12 @@ -327,13 +341,14 @@ may change without deprecation warnings. The old private name is deprecated, but will be available until the API changes. +.. c:function:: int _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) + :no-typesetting: + .. c:function:: int PyUnstable_Code_SetExtra(PyObject *code, Py_ssize_t index, void *extra) Set the extra data stored under the given index to *extra*. Return 0 on success. Set an exception and return -1 on failure. - .. index:: single: _PyCode_SetExtra (C function) - .. versionadded:: 3.6 as ``_PyCode_SetExtra`` .. versionchanged:: 3.12 diff --git a/Doc/c-api/codec.rst b/Doc/c-api/codec.rst index 8ae5c4fecd6..35ee048bd5f 100644 --- a/Doc/c-api/codec.rst +++ b/Doc/c-api/codec.rst @@ -7,7 +7,7 @@ Codec registry and support functions Register a new codec search function. - As side effect, this tries to load the :mod:`!encodings` package, if not yet + As a side effect, this tries to load the :mod:`!encodings` package, if not yet done, to make sure that it is always first in the list of search functions. .. c:function:: int PyCodec_Unregister(PyObject *search_function) @@ -39,7 +39,7 @@ Codec registry and support functions *object* is passed through the decoder function found for the given *encoding* using the error handling method defined by *errors*. *errors* may be ``NULL`` to use the default method defined for the codec. Raises a - :exc:`LookupError` if no encoder can be found. + :exc:`LookupError` if no decoder can be found. Codec lookup API @@ -129,3 +129,13 @@ Registry API for Unicode encoding error handlers Replace the unicode encode error with ``\N{...}`` escapes. .. versionadded:: 3.5 + + +Codec utility variables +----------------------- + +.. c:var:: const char *Py_hexdigits + + A string constant containing the lowercase hexadecimal digits: ``"0123456789abcdef"``. + + .. versionadded:: 3.3 diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index aa91b85d07f..629312bd771 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -16,7 +16,7 @@ Complex Number Objects The complex number value, using the C :c:type:`Py_complex` representation. - .. deprecated-removed:: next 3.20 + .. deprecated-removed:: 3.15 3.20 Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex` to convert a Python complex number to/from the C :c:type:`Py_complex` @@ -82,7 +82,7 @@ Complex Number Objects .. c:type:: Py_complex - This C structure defines export format for a Python complex + This C structure defines an export format for a Python complex number object. .. c:member:: double real diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index 880f7b15ce6..3f38411a52d 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -109,11 +109,21 @@ Other Objects descriptor.rst slice.rst memoryview.rst + picklebuffer.rst weakref.rst capsule.rst + sentinel.rst frame.rst gen.rst coro.rst contextvars.rst - datetime.rst typehints.rst + + +C API for extension modules +=========================== + +.. toctree:: + + curses.rst + datetime.rst diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index c92ef4c653a..f7c8ef8b22b 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -41,7 +41,7 @@ The return value (*rv*) for these functions should be interpreted as follows: ``rv + 1`` bytes would have been needed to succeed. ``str[size-1]`` is ``'\0'`` in this case. -* When ``rv < 0``, "something bad happened." ``str[size-1]`` is ``'\0'`` in +* When ``rv < 0``, the output conversion failed and ``str[size-1]`` is ``'\0'`` in this case too, but the rest of *str* is undefined. The exact cause of the error depends on the underlying platform. @@ -105,7 +105,7 @@ The following functions provide locale-independent string to number conversions. If ``s`` represents a value that is too large to store in a float (for example, ``"1e500"`` is such a string on many platforms) then - if ``overflow_exception`` is ``NULL`` return ``Py_INFINITY`` (with + if ``overflow_exception`` is ``NULL`` return :c:macro:`!INFINITY` (with an appropriate sign) and don't set any exception. Otherwise, ``overflow_exception`` must point to a Python exception object; raise that exception and return ``-1.0``. In both cases, set @@ -128,22 +128,46 @@ The following functions provide locale-independent string to number conversions. must be 0 and is ignored. The ``'r'`` format code specifies the standard :func:`repr` format. - *flags* can be zero or more of the values ``Py_DTSF_SIGN``, - ``Py_DTSF_ADD_DOT_0``, or ``Py_DTSF_ALT``, or-ed together: + *flags* can be zero or more of the following values or-ed together: - * ``Py_DTSF_SIGN`` means to always precede the returned string with a sign - character, even if *val* is non-negative. + .. c:namespace:: NULL - * ``Py_DTSF_ADD_DOT_0`` means to ensure that the returned string will not look - like an integer. + .. c:macro:: Py_DTSF_SIGN - * ``Py_DTSF_ALT`` means to apply "alternate" formatting rules. See the - documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for - details. + Always precede the returned string with a sign + character, even if *val* is non-negative. - If *ptype* is non-``NULL``, then the value it points to will be set to one of - ``Py_DTST_FINITE``, ``Py_DTST_INFINITE``, or ``Py_DTST_NAN``, signifying that - *val* is a finite number, an infinite number, or not a number, respectively. + .. c:macro:: Py_DTSF_ADD_DOT_0 + + Ensure that the returned string will not look like an integer. + + .. c:macro:: Py_DTSF_ALT + + Apply "alternate" formatting rules. + See the documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for + details. + + .. c:macro:: Py_DTSF_NO_NEG_0 + + Negative zero is converted to positive zero. + + .. versionadded:: 3.11 + + If *ptype* is non-``NULL``, then the value it points to will be set to one + of the following constants depending on the type of *val*: + + .. list-table:: + :header-rows: 1 + :align: left + + * - *\*ptype* + - type of *val* + * - .. c:macro:: Py_DTST_FINITE + - finite number + * - .. c:macro:: Py_DTST_INFINITE + - infinite number + * - .. c:macro:: Py_DTST_NAN + - not a number The return value is a pointer to *buffer* with the converted string or ``NULL`` if the conversion failed. The caller is responsible for freeing the @@ -152,13 +176,85 @@ The following functions provide locale-independent string to number conversions. .. versionadded:: 3.1 -.. c:function:: int PyOS_stricmp(const char *s1, const char *s2) +.. c:function:: int PyOS_mystricmp(const char *str1, const char *str2) + int PyOS_mystrnicmp(const char *str1, const char *str2, Py_ssize_t size) - Case insensitive comparison of strings. The function works almost - identically to :c:func:`!strcmp` except that it ignores the case. + Case insensitive comparison of strings. These functions work almost + identically to :c:func:`!strcmp` and :c:func:`!strncmp` (respectively), + except that they ignore the case of ASCII characters. + + Return ``0`` if the strings are equal, a negative value if *str1* sorts + lexicographically before *str2*, or a positive value if it sorts after. + + In the *str1* or *str2* arguments, a NUL byte marks the end of the string. + For :c:func:`!PyOS_mystrnicmp`, the *size* argument gives the maximum size + of the string, as if NUL was present at the index given by *size*. + + These functions do not use the locale. -.. c:function:: int PyOS_strnicmp(const char *s1, const char *s2, Py_ssize_t size) +.. c:function:: int PyOS_stricmp(const char *str1, const char *str2) + int PyOS_strnicmp(const char *str1, const char *str2, Py_ssize_t size) - Case insensitive comparison of strings. The function works almost - identically to :c:func:`!strncmp` except that it ignores the case. + Case insensitive comparison of strings. + + On Windows, these are aliases of :c:func:`!stricmp` and :c:func:`!strnicmp`, + respectively. + + On other platforms, they are aliases of :c:func:`PyOS_mystricmp` and + :c:func:`PyOS_mystrnicmp`, respectively. + + +Character classification and conversion +======================================= + +The following macros provide locale-independent (unlike the C standard library +``ctype.h``) character classification and conversion. +The argument must be a signed or unsigned :c:expr:`char`. + + +.. c:macro:: Py_ISALNUM(c) + + Return true if the character *c* is an alphanumeric character. + + +.. c:macro:: Py_ISALPHA(c) + + Return true if the character *c* is an alphabetic character (``a-z`` and ``A-Z``). + + +.. c:macro:: Py_ISDIGIT(c) + + Return true if the character *c* is a decimal digit (``0-9``). + + +.. c:macro:: Py_ISLOWER(c) + + Return true if the character *c* is a lowercase ASCII letter (``a-z``). + + +.. c:macro:: Py_ISUPPER(c) + + Return true if the character *c* is an uppercase ASCII letter (``A-Z``). + + +.. c:macro:: Py_ISSPACE(c) + + Return true if the character *c* is a whitespace character (space, tab, + carriage return, newline, vertical tab, or form feed). + + +.. c:macro:: Py_ISXDIGIT(c) + + Return true if the character *c* is a hexadecimal digit (``0-9``, ``a-f``, and + ``A-F``). + + +.. c:macro:: Py_TOLOWER(c) + + Return the lowercase equivalent of the character *c*. + + +.. c:macro:: Py_TOUPPER(c) + + Return the uppercase equivalent of the character *c*. diff --git a/Doc/c-api/curses.rst b/Doc/c-api/curses.rst new file mode 100644 index 00000000000..5a1697c43cc --- /dev/null +++ b/Doc/c-api/curses.rst @@ -0,0 +1,138 @@ +.. highlight:: c + +Curses C API +------------ + +:mod:`curses` exposes a small C interface for extension modules. +Consumers must include the header file :file:`py_curses.h` (which is not +included by default by :file:`Python.h`) and :c:func:`import_curses` must +be invoked, usually as part of the module initialisation function, to populate +:c:var:`PyCurses_API`. + +.. warning:: + + Neither the C API nor the pure Python :mod:`curses` module are compatible + with subinterpreters. + +.. c:macro:: import_curses() + + Import the curses C API. The macro does not need a semi-colon to be called. + + On success, populate the :c:var:`PyCurses_API` pointer. + + On failure, set :c:var:`PyCurses_API` to NULL and set an exception. + The caller must check if an error occurred via :c:func:`PyErr_Occurred`: + + .. code-block:: + + import_curses(); // semi-colon is optional but recommended + if (PyErr_Occurred()) { /* cleanup */ } + + +.. c:var:: void **PyCurses_API + + Dynamically allocated object containing the curses C API. + This variable is only available once :c:macro:`import_curses` succeeds. + + ``PyCurses_API[0]`` corresponds to :c:data:`PyCursesWindow_Type`. + + ``PyCurses_API[1]``, ``PyCurses_API[2]``, and ``PyCurses_API[3]`` + are pointers to predicate functions of type ``int (*)(void)``. + + When called, these predicates return whether :func:`curses.setupterm`, + :func:`curses.initscr`, and :func:`curses.start_color` have been called + respectively. + + See also the convenience macros :c:macro:`PyCursesSetupTermCalled`, + :c:macro:`PyCursesInitialised`, and :c:macro:`PyCursesInitialisedColor`. + + .. note:: + + The number of entries in this structure is subject to changes. + Consider using :c:macro:`PyCurses_API_pointers` to check if + new fields are available or not. + + +.. c:macro:: PyCurses_API_pointers + + The number of accessible fields (``4``) in :c:var:`PyCurses_API`. + This number is incremented whenever new fields are added. + + +.. c:var:: PyTypeObject PyCursesWindow_Type + + The :ref:`heap type ` corresponding to :class:`curses.window`. + + +.. c:function:: int PyCursesWindow_Check(PyObject *op) + + Return true if *op* is a :class:`curses.window` instance, false otherwise. + + +The following macros are convenience macros expanding into C statements. +In particular, they can only be used as ``macro;`` or ``macro``, but not +``macro()`` or ``macro();``. + +.. c:macro:: PyCursesSetupTermCalled + + Macro checking if :func:`curses.setupterm` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_setupterm_called = (predicate_t)PyCurses_API[1]; + if (!was_setupterm_called()) { + return NULL; + } + } + + +.. c:macro:: PyCursesInitialised + + Macro checking if :func:`curses.initscr` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_initscr_called = (predicate_t)PyCurses_API[2]; + if (!was_initscr_called()) { + return NULL; + } + } + + +.. c:macro:: PyCursesInitialisedColor + + Macro checking if :func:`curses.start_color` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_start_color_called = (predicate_t)PyCurses_API[3]; + if (!was_start_color_called()) { + return NULL; + } + } + + +Internal data +------------- + +The following objects are exposed by the C API but should be considered +internal-only. + +.. c:macro:: PyCurses_CAPSULE_NAME + + Name of the curses capsule to pass to :c:func:`PyCapsule_Import`. + + Internal usage only. Use :c:macro:`import_curses` instead. + diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index d2d4d5309c7..d7b4e116c49 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -8,11 +8,51 @@ DateTime Objects Various date and time objects are supplied by the :mod:`datetime` module. Before using any of these functions, the header file :file:`datetime.h` must be included in your source (note that this is not included by :file:`Python.h`), -and the macro :c:macro:`!PyDateTime_IMPORT` must be invoked, usually as part of +and the macro :c:macro:`PyDateTime_IMPORT` must be invoked, usually as part of the module initialisation function. The macro puts a pointer to a C structure -into a static variable, :c:data:`!PyDateTimeAPI`, that is used by the following +into a static variable, :c:data:`PyDateTimeAPI`, that is used by the following macros. +.. c:macro:: PyDateTime_IMPORT() + + Import the datetime C API. + + On success, populate the :c:var:`PyDateTimeAPI` pointer. + On failure, set :c:var:`PyDateTimeAPI` to ``NULL`` and set an exception. + The caller must check if an error occurred via :c:func:`PyErr_Occurred`: + + .. code-block:: + + PyDateTime_IMPORT; + if (PyErr_Occurred()) { /* cleanup */ } + + .. warning:: + + This is not compatible with subinterpreters. + + .. versionchanged:: 3.15 + + This macro is now thread safe. + +.. c:type:: PyDateTime_CAPI + + Structure containing the fields for the datetime C API. + + The fields of this structure are private and subject to change. + + Do not use this directly; prefer ``PyDateTime_*`` APIs instead. + +.. c:var:: PyDateTime_CAPI *PyDateTimeAPI + + Dynamically allocated object containing the datetime C API. + + This variable is only available once :c:macro:`PyDateTime_IMPORT` succeeds. + + .. versionchanged:: 3.15 + + This variable should not be accessed directly as direct access is not thread-safe. + Use :c:func:`PyDateTime_IMPORT` instead. + .. c:type:: PyDateTime_Date This subtype of :c:type:`PyObject` represents a Python date object. @@ -46,7 +86,7 @@ macros. .. c:var:: PyTypeObject PyDateTime_DeltaType - This instance of :c:type:`PyTypeObject` represents Python type for + This instance of :c:type:`PyTypeObject` represents the Python type for the difference between two datetime values; it is the same object as :class:`datetime.timedelta` in the Python layer. @@ -325,3 +365,16 @@ Macros for the convenience of modules implementing the DB API: Create and return a new :class:`datetime.date` object given an argument tuple suitable for passing to :meth:`datetime.date.fromtimestamp`. + + +Internal data +------------- + +The following symbols are exposed by the C API but should be considered +internal-only. + +.. c:macro:: PyDateTime_CAPSULE_NAME + + Name of the datetime capsule to pass to :c:func:`PyCapsule_Import`. + + Internal usage only. Use :c:macro:`PyDateTime_IMPORT` instead. diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index b32c113e5f0..539c4610ce4 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -8,33 +8,200 @@ Descriptor Objects "Descriptors" are objects that describe some attribute of an object. They are found in the dictionary of type objects. -.. XXX document these! - -.. c:var:: PyTypeObject PyProperty_Type - - The type object for the built-in descriptor types. - - .. c:function:: PyObject* PyDescr_NewGetSet(PyTypeObject *type, struct PyGetSetDef *getset) + Create a new get-set descriptor for extension type *type* from the + :c:type:`PyGetSetDef` structure *getset*. -.. c:function:: PyObject* PyDescr_NewMember(PyTypeObject *type, struct PyMemberDef *meth) + Get-set descriptors expose attributes implemented by C getter and setter + functions rather than stored directly in the instance. This is the same kind + of descriptor created for entries in :c:member:`~PyTypeObject.tp_getset`, and + it appears in Python as a :class:`types.GetSetDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. + +.. c:function:: PyObject* PyDescr_NewMember(PyTypeObject *type, struct PyMemberDef *member) + + Create a new member descriptor for extension type *type* from the + :c:type:`PyMemberDef` structure *member*. + + Member descriptors expose fields in the type's C struct as Python + attributes. This is the same kind of descriptor created for entries in + :c:member:`~PyTypeObject.tp_members`, and it appears in Python as a + :class:`types.MemberDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. + +.. c:var:: PyTypeObject PyMemberDescr_Type + + The type object for member descriptor objects created from + :c:type:`PyMemberDef` structures. These descriptors expose fields of a + C struct as attributes on a type, and correspond + to :class:`types.MemberDescriptorType` objects in Python. + + + +.. c:var:: PyTypeObject PyGetSetDescr_Type + + The type object for get/set descriptor objects created from + :c:type:`PyGetSetDef` structures. These descriptors implement attributes + whose value is computed by C getter and setter functions, and are used + for many built-in type attributes. They correspond to + :class:`types.GetSetDescriptorType` objects in Python. .. c:function:: PyObject* PyDescr_NewMethod(PyTypeObject *type, struct PyMethodDef *meth) + Create a new method descriptor for extension type *type* from the + :c:type:`PyMethodDef` structure *meth*. -.. c:function:: PyObject* PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *wrapper, void *wrapped) + Method descriptors expose C functions as methods on a type. This is the same + kind of descriptor created for entries in + :c:member:`~PyTypeObject.tp_methods`, and it appears in Python as a + :class:`types.MethodDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. + +.. c:var:: PyTypeObject PyMethodDescr_Type + + The type object for method descriptor objects created from + :c:type:`PyMethodDef` structures. These descriptors expose C functions as + methods on a type, and correspond to :class:`types.MethodDescriptorType` + objects in Python. + + +.. c:struct:: wrapperbase + + Describes a slot wrapper used by :c:func:`PyDescr_NewWrapper`. + + Each ``wrapperbase`` record stores the Python-visible name and metadata for a + special method implemented by a type slot, together with the wrapper + function used to adapt that slot to Python's calling convention. + +.. c:function:: PyObject* PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *base, void *wrapped) + + Create a new wrapper descriptor for extension type *type* from the + :c:struct:`wrapperbase` structure *base* and the wrapped slot function + pointer + *wrapped*. + + Wrapper descriptors expose special methods implemented by type slots. This + is the same kind of descriptor that CPython creates for slot-based special + methods such as ``__repr__`` or ``__add__``, and it appears in Python as a + :class:`types.WrapperDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. + +.. c:var:: PyTypeObject PyWrapperDescr_Type + + The type object for wrapper descriptor objects created by + :c:func:`PyDescr_NewWrapper` and :c:func:`PyWrapper_New`. Wrapper + descriptors are used internally to expose special methods implemented + via wrapper structures, and appear in Python as + :class:`types.WrapperDescriptorType` objects. .. c:function:: PyObject* PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method) + Create a new class method descriptor for extension type *type* from the + :c:type:`PyMethodDef` structure *method*. + + Class method descriptors expose C methods that receive the class rather than + an instance when accessed. This is the same kind of descriptor created for + ``METH_CLASS`` entries in :c:member:`~PyTypeObject.tp_methods`, and it + appears in Python as a :class:`types.ClassMethodDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. .. c:function:: int PyDescr_IsData(PyObject *descr) - Return non-zero if the descriptor objects *descr* describes a data attribute, or + Return non-zero if the descriptor object *descr* describes a data attribute, or ``0`` if it describes a method. *descr* must be a descriptor object; there is no error checking. -.. c:function:: PyObject* PyWrapper_New(PyObject *, PyObject *) +.. c:function:: PyObject* PyWrapper_New(PyObject *d, PyObject *self) + + Create a new bound wrapper object from the wrapper descriptor *d* and the + instance *self*. + + This is the bound form of a wrapper descriptor created by + :c:func:`PyDescr_NewWrapper`. CPython creates these objects when a slot + wrapper is accessed through an instance, and they appear in Python as + :class:`types.MethodWrapperType` objects. + + On success, return a :term:`strong reference` to the wrapper object. Return + ``NULL`` with an exception set on failure. + +.. c:macro:: PyDescr_COMMON + + This is a macro including the common fields for a + descriptor object. + + This was included in Python's C API by mistake; do not use it in extensions. + For creating custom descriptor objects, create a class implementing the + descriptor protocol (:c:member:`~PyTypeObject.tp_descr_get` and + :c:member:`~PyTypeObject.tp_descr_set`). + + .. soft-deprecated:: 3.15 + + +Built-in descriptors +^^^^^^^^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PyProperty_Type + + The type object for property objects. This is the same object as + :class:`property` in the Python layer. + + +.. c:var:: PyTypeObject PySuper_Type + + The type object for super objects. This is the same object as + :class:`super` in the Python layer. + + +.. c:var:: PyTypeObject PyClassMethod_Type + + The type of class method objects. This is the same object as + :class:`classmethod` in the Python layer. + + +.. c:var:: PyTypeObject PyClassMethodDescr_Type + + The type object for C-level class method descriptor objects. + This is the type of the descriptors created for :func:`classmethod` defined + in C extension types, and corresponds to + :class:`types.ClassMethodDescriptorType` objects in Python. + + +.. c:function:: PyObject *PyClassMethod_New(PyObject *callable) + + Create a new :class:`classmethod` object wrapping *callable*. + *callable* must be a callable object and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to a new class + method descriptor. On failure, this function returns ``NULL`` with an + exception set. + + +.. c:var:: PyTypeObject PyStaticMethod_Type + + The type of static method objects. This is the same object as + :class:`staticmethod` in the Python layer. + + +.. c:function:: PyObject *PyStaticMethod_New(PyObject *callable) + + Create a new :class:`staticmethod` object wrapping *callable*. + *callable* must be a callable object and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to a new static + method descriptor. On failure, this function returns ``NULL`` with an + exception set. + diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 0fbe26b56c0..a2a0d0d8065 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -2,7 +2,7 @@ .. _dictobjects: -Dictionary Objects +Dictionary objects ------------------ .. index:: pair: object; dictionary @@ -42,18 +42,48 @@ Dictionary Objects enforces read-only behavior. This is normally used to create a view to prevent modification of the dictionary for non-dynamic class types. + The first argument can be a :class:`dict`, a :class:`frozendict`, or a + mapping. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + + +.. c:var:: PyTypeObject PyDictProxy_Type + + The type object for mapping proxy objects created by + :c:func:`PyDictProxy_New` and for the read-only ``__dict__`` attribute + of many built-in types. A :c:type:`PyDictProxy_Type` instance provides a + dynamic, read-only view of an underlying dictionary: changes to the + underlying dictionary are reflected in the proxy, but the proxy itself + does not support mutation operations. This corresponds to + :class:`types.MappingProxyType` in Python. + .. c:function:: void PyDict_Clear(PyObject *p) Empty an existing dictionary of all key-value pairs. + Do nothing if the argument is not a :class:`dict` or a :class:`!dict` + subclass. + .. c:function:: int PyDict_Contains(PyObject *p, PyObject *key) - Determine if dictionary *p* contains *key*. If an item in *p* is matches + Determine if dictionary *p* contains *key*. If an item in *p* matches *key*, return ``1``, otherwise return ``0``. On error, return ``-1``. This is equivalent to the Python expression ``key in p``. + The first argument can be a :class:`dict` or a :class:`frozendict`. + + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: int PyDict_ContainsString(PyObject *p, const char *key) @@ -61,14 +91,18 @@ Dictionary Objects :c:expr:`const char*` UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. + The first argument can be a :class:`dict` or a :class:`frozendict`. + .. versionadded:: 3.13 + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: PyObject* PyDict_Copy(PyObject *p) Return a new dictionary that contains the same key-value pairs as *p*. - .. c:function:: int PyDict_SetItem(PyObject *p, PyObject *key, PyObject *val) Insert *val* into the dictionary *p* with a key of *key*. *key* must be @@ -76,6 +110,11 @@ Dictionary Objects ``0`` on success or ``-1`` on failure. This function *does not* steal a reference to *val*. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. c:function:: int PyDict_SetItemString(PyObject *p, const char *key, PyObject *val) @@ -91,6 +130,11 @@ Dictionary Objects If *key* is not in the dictionary, :exc:`KeyError` is raised. Return ``0`` on success or ``-1`` on failure. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. c:function:: int PyDict_DelItemString(PyObject *p, const char *key) @@ -109,8 +153,18 @@ Dictionary Objects * If the key is missing, set *\*result* to ``NULL`` and return ``0``. * On error, raise an exception and return ``-1``. + The first argument can be a :class:`dict` or a :class:`frozendict`. + + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. versionadded:: 3.13 + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + See also the :c:func:`PyObject_GetItem` function. @@ -120,16 +174,28 @@ Dictionary Objects has a key *key*. Return ``NULL`` if the key *key* is missing *without* setting an exception. + The first argument can be a :class:`dict` or a :class:`frozendict`. + .. note:: Exceptions that occur while this calls :meth:`~object.__hash__` and :meth:`~object.__eq__` methods are silently ignored. Prefer the :c:func:`PyDict_GetItemWithError` function instead. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_GetItemRef`, which + returns a :term:`strong reference`. + .. versionchanged:: 3.10 Calling this API without an :term:`attached thread state` had been allowed for historical reason. It is no longer allowed. + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: PyObject* PyDict_GetItemWithError(PyObject *p, PyObject *key) @@ -138,6 +204,16 @@ Dictionary Objects occurred. Return ``NULL`` **without** an exception set if the key wasn't present. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_GetItemRef`, which + returns a :term:`strong reference`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: PyObject* PyDict_GetItemString(PyObject *p, const char *key) @@ -153,6 +229,16 @@ Dictionary Objects Prefer using the :c:func:`PyDict_GetItemWithError` function with your own :c:func:`PyUnicode_FromString` *key* instead. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_GetItemStringRef`, + which returns a :term:`strong reference`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: int PyDict_GetItemStringRef(PyObject *p, const char *key, PyObject **result) @@ -162,6 +248,9 @@ Dictionary Objects .. versionadded:: 3.13 + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: PyObject* PyDict_SetDefault(PyObject *p, PyObject *key, PyObject *defaultobj) @@ -173,6 +262,14 @@ Dictionary Objects .. versionadded:: 3.4 + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_SetDefaultRef`, + which returns a :term:`strong reference`. + + .. c:function:: int PyDict_SetDefaultRef(PyObject *p, PyObject *key, PyObject *default_value, PyObject **result) @@ -192,13 +289,18 @@ Dictionary Objects These may refer to the same object: in that case you hold two separate references to it. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. versionadded:: 3.13 .. c:function:: int PyDict_Pop(PyObject *p, PyObject *key, PyObject **result) Remove *key* from dictionary *p* and optionally return the removed value. - Do not raise :exc:`KeyError` if the key missing. + Do not raise :exc:`KeyError` if the key is missing. - If the key is present, set *\*result* to a new reference to the removed value if *result* is not ``NULL``, and return ``1``. @@ -207,7 +309,12 @@ Dictionary Objects - On error, raise an exception and return ``-1``. Similar to :meth:`dict.pop`, but without the default value and - not raising :exc:`KeyError` if the key missing. + not raising :exc:`KeyError` if the key is missing. + + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. .. versionadded:: 3.13 @@ -225,17 +332,32 @@ Dictionary Objects Return a :c:type:`PyListObject` containing all the items from the dictionary. + The first argument can be a :class:`dict` or a :class:`frozendict`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: PyObject* PyDict_Keys(PyObject *p) Return a :c:type:`PyListObject` containing all the keys from the dictionary. + The first argument can be a :class:`dict` or a :class:`frozendict`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: PyObject* PyDict_Values(PyObject *p) Return a :c:type:`PyListObject` containing all the values from the dictionary *p*. + The first argument can be a :class:`dict` or a :class:`frozendict`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: Py_ssize_t PyDict_Size(PyObject *p) @@ -244,6 +366,19 @@ Dictionary Objects Return the number of items in the dictionary. This is equivalent to ``len(p)`` on a dictionary. + The argument can be a :class:`dict` or a :class:`frozendict`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + + +.. c:function:: Py_ssize_t PyDict_GET_SIZE(PyObject *p) + + Similar to :c:func:`PyDict_Size`, but without error checking. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: int PyDict_Next(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) @@ -258,6 +393,8 @@ Dictionary Objects value represents offsets within the internal dictionary structure, and since the structure is sparse, the offsets are not consecutive. + The first argument can be a :class:`dict` or a :class:`frozendict`. + For example:: PyObject *key, *value; @@ -291,7 +428,7 @@ Dictionary Objects } The function is not thread-safe in the :term:`free-threaded ` - build without external synchronization. You can use + build without external synchronization for a mutable :class:`dict`. You can use :c:macro:`Py_BEGIN_CRITICAL_SECTION` to lock the dictionary while iterating over it:: @@ -301,6 +438,8 @@ Dictionary Objects } Py_END_CRITICAL_SECTION(); + The function is thread-safe on a :class:`frozendict`. + .. note:: On the free-threaded build, this function can be used safely inside a @@ -311,6 +450,9 @@ Dictionary Objects :term:`strong reference ` (for example, using :c:func:`Py_NewRef`). + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: int PyDict_Merge(PyObject *a, PyObject *b, int override) Iterate over mapping object *b* adding key-value pairs to dictionary *a*. @@ -320,6 +462,13 @@ Dictionary Objects only be added if there is not a matching key in *a*. Return ``0`` on success or ``-1`` if an exception was raised. + .. note:: + + In the :term:`free-threaded build`, when *b* is a + :class:`dict` (with the standard iterator), both *a* and *b* are locked + for the duration of the operation. When *b* is a non-dict mapping, only + *a* is locked; *b* may be concurrently modified by another thread. + .. c:function:: int PyDict_Update(PyObject *a, PyObject *b) @@ -329,6 +478,13 @@ Dictionary Objects argument has no "keys" attribute. Return ``0`` on success or ``-1`` if an exception was raised. + .. note:: + + In the :term:`free-threaded build`, when *b* is a + :class:`dict` (with the standard iterator), both *a* and *b* are locked + for the duration of the operation. When *b* is a non-dict mapping, only + *a* is locked; *b* may be concurrently modified by another thread. + .. c:function:: int PyDict_MergeFromSeq2(PyObject *a, PyObject *seq2, int override) @@ -344,6 +500,13 @@ Dictionary Objects if override or key not in a: a[key] = value + .. note:: + + In the :term:`free-threaded ` build, only *a* is locked. + The iteration over *seq2* is not synchronized; *seq2* may be concurrently + modified by another thread. + + .. c:function:: int PyDict_AddWatcher(PyDict_WatchCallback callback) Register *callback* as a dictionary watcher. Return a non-negative integer @@ -351,6 +514,13 @@ Dictionary Objects of error (e.g. no more watcher IDs available), return ``-1`` and set an exception. + .. note:: + + This function is not internally synchronized. In the + :term:`free-threaded ` build, callers should ensure no + concurrent calls to :c:func:`PyDict_AddWatcher` or + :c:func:`PyDict_ClearWatcher` are in progress. + .. versionadded:: 3.12 .. c:function:: int PyDict_ClearWatcher(int watcher_id) @@ -359,6 +529,13 @@ Dictionary Objects :c:func:`PyDict_AddWatcher`. Return ``0`` on success, ``-1`` on error (e.g. if the given *watcher_id* was never registered.) + .. note:: + + This function is not internally synchronized. In the + :term:`free-threaded ` build, callers should ensure no + concurrent calls to :c:func:`PyDict_AddWatcher` or + :c:func:`PyDict_ClearWatcher` are in progress. + .. versionadded:: 3.12 .. c:function:: int PyDict_Watch(int watcher_id, PyObject *dict) @@ -426,3 +603,189 @@ Dictionary Objects it before returning. .. versionadded:: 3.12 + + +Dictionary view objects +^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: int PyDictViewSet_Check(PyObject *op) + + Return true if *op* is a view of a set inside a dictionary. This is currently + equivalent to :c:expr:`PyDictKeys_Check(op) || PyDictItems_Check(op)`. This + function always succeeds. + + +.. c:var:: PyTypeObject PyDictKeys_Type + + Type object for a view of dictionary keys. In Python, this is the type of + the object returned by :meth:`dict.keys`. + + +.. c:function:: int PyDictKeys_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary keys view. This function + always succeeds. + + +.. c:var:: PyTypeObject PyDictValues_Type + + Type object for a view of dictionary values. In Python, this is the type of + the object returned by :meth:`dict.values`. + + +.. c:function:: int PyDictValues_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary values view. This function + always succeeds. + + +.. c:var:: PyTypeObject PyDictItems_Type + + Type object for a view of dictionary items. In Python, this is the type of + the object returned by :meth:`dict.items`. + + +.. c:function:: int PyDictItems_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary items view. This function + always succeeds. + + +Frozen dictionary objects +^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 3.15 + + +.. c:var:: PyTypeObject PyFrozenDict_Type + + This instance of :c:type:`PyTypeObject` represents the Python frozen + dictionary type. + This is the same object as :class:`frozendict` in the Python layer. + + +.. c:function:: int PyAnyDict_Check(PyObject *p) + + Return true if *p* is a :class:`dict` object, a :class:`frozendict` object, + or an instance of a subtype of the :class:`!dict` or :class:`!frozendict` + type. + This function always succeeds. + + +.. c:function:: int PyAnyDict_CheckExact(PyObject *p) + + Return true if *p* is a :class:`dict` object or a :class:`frozendict` object, + but not an instance of a subtype of the :class:`!dict` or + :class:`!frozendict` type. + This function always succeeds. + + +.. c:function:: int PyFrozenDict_Check(PyObject *p) + + Return true if *p* is a :class:`frozendict` object or an instance of a + subtype of the :class:`!frozendict` type. + This function always succeeds. + + +.. c:function:: int PyFrozenDict_CheckExact(PyObject *p) + + Return true if *p* is a :class:`frozendict` object, but not an instance of a + subtype of the :class:`!frozendict` type. + This function always succeeds. + + +.. c:function:: PyObject* PyFrozenDict_New(PyObject *iterable) + + Return a new :class:`frozendict` from an iterable, or ``NULL`` on failure + with an exception set. + + Create an empty dictionary if *iterable* is ``NULL``. + + +Ordered dictionaries +^^^^^^^^^^^^^^^^^^^^ + +Python's C API provides interface for :class:`collections.OrderedDict` from C. +Since Python 3.7, dictionaries are ordered by default, so there is usually +little need for these functions; prefer ``PyDict*`` where possible. + + +.. c:var:: PyTypeObject PyODict_Type + + Type object for ordered dictionaries. This is the same object as + :class:`collections.OrderedDict` in the Python layer. + + +.. c:function:: int PyODict_Check(PyObject *od) + + Return true if *od* is an ordered dictionary object or an instance of a + subtype of the :class:`~collections.OrderedDict` type. This function + always succeeds. + + +.. c:function:: int PyODict_CheckExact(PyObject *od) + + Return true if *od* is an ordered dictionary object, but not an instance of + a subtype of the :class:`~collections.OrderedDict` type. + This function always succeeds. + + +.. c:var:: PyTypeObject PyODictKeys_Type + + Analogous to :c:type:`PyDictKeys_Type` for ordered dictionaries. + + +.. c:var:: PyTypeObject PyODictValues_Type + + Analogous to :c:type:`PyDictValues_Type` for ordered dictionaries. + + +.. c:var:: PyTypeObject PyODictItems_Type + + Analogous to :c:type:`PyDictItems_Type` for ordered dictionaries. + + +.. c:function:: PyObject *PyODict_New(void) + + Return a new empty ordered dictionary, or ``NULL`` on failure. + + This is analogous to :c:func:`PyDict_New`. + + +.. c:function:: int PyODict_SetItem(PyObject *od, PyObject *key, PyObject *value) + + Insert *value* into the ordered dictionary *od* with a key of *key*. + Return ``0`` on success or ``-1`` with an exception set on failure. + + This is analogous to :c:func:`PyDict_SetItem`. + + +.. c:function:: int PyODict_DelItem(PyObject *od, PyObject *key) + + Remove the entry in the ordered dictionary *od* with key *key*. + Return ``0`` on success or ``-1`` with an exception set on failure. + + This is analogous to :c:func:`PyDict_DelItem`. + + +These are :term:`soft deprecated` aliases to ``PyDict`` APIs: + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * ``PyODict`` + * ``PyDict`` + * * .. c:macro:: PyODict_GetItem(od, key) + * :c:func:`PyDict_GetItem` + * * .. c:macro:: PyODict_GetItemWithError(od, key) + * :c:func:`PyDict_GetItemWithError` + * * .. c:macro:: PyODict_GetItemString(od, key) + * :c:func:`PyDict_GetItemString` + * * .. c:macro:: PyODict_Contains(od, key) + * :c:func:`PyDict_Contains` + * * .. c:macro:: PyODict_Size(od) + * :c:func:`PyDict_Size` + * * .. c:macro:: PyODict_SIZE(od) + * :c:func:`PyDict_GET_SIZE` diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 3ff4631a8e5..7a07818b7b4 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -309,6 +309,14 @@ For convenience, some of these functions will always return a .. versionadded:: 3.4 +.. c:function:: void PyErr_RangedSyntaxLocationObject(PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset) + + Similar to :c:func:`PyErr_SyntaxLocationObject`, but also sets the + *end_lineno* and *end_col_offset* information for the current exception. + + .. versionadded:: 3.10 + + .. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string @@ -331,6 +339,23 @@ For convenience, some of these functions will always return a use. +.. c:function:: PyObject *PyErr_ProgramTextObject(PyObject *filename, int lineno) + + Get the source line in *filename* at line *lineno*. *filename* should be a + Python :class:`str` object. + + On success, this function returns a Python string object with the found line. + On failure, this function returns ``NULL`` without an exception set. + + +.. c:function:: PyObject *PyErr_ProgramText(const char *filename, int lineno) + + Similar to :c:func:`PyErr_ProgramTextObject`, but *filename* is a + :c:expr:`const char *`, which is decoded with the + :term:`filesystem encoding and error handler`, instead of a + Python object reference. + + Issuing warnings ================ @@ -394,6 +419,15 @@ an error value). .. versionadded:: 3.2 +.. c:function:: int PyErr_WarnExplicitFormat(PyObject *category, const char *filename, int lineno, const char *module, PyObject *registry, const char *format, ...) + + Similar to :c:func:`PyErr_WarnExplicit`, but uses + :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is + an ASCII-encoded string. + + .. versionadded:: 3.2 + + .. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) Function similar to :c:func:`PyErr_WarnFormat`, but *category* is @@ -639,28 +673,51 @@ Signal Handling single: SIGINT (C macro) single: KeyboardInterrupt (built-in exception) - This function interacts with Python's signal handling. + Handle external interruptions, such as signals or activating a debugger, + whose processing has been delayed until it is safe + to run Python code and/or raise exceptions. - If the function is called from the main thread and under the main Python - interpreter, it checks whether a signal has been sent to the processes - and if so, invokes the corresponding signal handler. If the :mod:`signal` - module is supported, this can invoke a signal handler written in Python. + For example, pressing :kbd:`Ctrl-C` causes a terminal to send the + :py:data:`signal.SIGINT` signal. + This function executes the corresponding Python signal handler, which, + by default, raises the :exc:`KeyboardInterrupt` exception. - The function attempts to handle all pending signals, and then returns ``0``. - However, if a Python signal handler raises an exception, the error - indicator is set and the function returns ``-1`` immediately (such that - other pending signals may not have been handled yet: they will be on the - next :c:func:`PyErr_CheckSignals()` invocation). + :c:func:`!PyErr_CheckSignals` should be called by long-running C code + frequently enough so that the response appears immediate to humans. - If the function is called from a non-main thread, or under a non-main - Python interpreter, it does nothing and returns ``0``. + Handlers invoked by this function currently include: - This function can be called by long-running C code that wants to - be interruptible by user requests (such as by pressing Ctrl-C). + - Signal handlers, including Python functions registered using + the :mod:`signal` module. - .. note:: - The default Python signal handler for :c:macro:`!SIGINT` raises the - :exc:`KeyboardInterrupt` exception. + Signal handlers are only run in the main thread of the main interpreter. + + (This is where the function got the name: originally, signals + were the only way to interrupt the interpreter.) + + - Running the garbage collector, if necessary. + + - Executing a pending :ref:`remote debugger ` script. + + - Raise the exception set by :c:func:`PyThreadState_SetAsyncExc`. + + If any handler raises an exception, immediately return ``-1`` with that + exception set. + Any remaining interruptions are left to be processed on the next + :c:func:`PyErr_CheckSignals()` invocation, if appropriate. + + If all handlers finish successfully, or there are no handlers to run, + return ``0``. + + .. versionchanged:: 3.12 + This function may now invoke the garbage collector. + + .. versionchanged:: 3.14 + This function may now execute a remote debugger script, if remote + debugging is enabled. + + .. versionchanged:: 3.15 + The exception set by :c:func:`PyThreadState_SetAsyncExc` is now raised. .. c:function:: void PyErr_SetInterrupt() @@ -759,9 +816,33 @@ Exception Classes Return :c:member:`~PyTypeObject.tp_name` of the exception class *ob*. +.. c:macro:: PyException_HEAD + + This is a macro including the base fields for an + exception object. + + This was included in Python's C API by mistake and is not designed for use + in extensions. For creating custom exception objects, use + :c:func:`PyErr_NewException` or otherwise create a class inheriting from + :c:data:`PyExc_BaseException`. + + .. soft-deprecated:: 3.15 + + Exception Objects ================= +.. c:function:: int PyExceptionInstance_Check(PyObject *op) + + Return true if *op* is an instance of :class:`BaseException`, false + otherwise. This function always succeeds. + + +.. c:macro:: PyExceptionInstance_Class(op) + + Equivalent to :c:func:`Py_TYPE(op) `. + + .. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) Return the traceback associated with the exception as a new reference, as @@ -939,6 +1020,9 @@ because the :ref:`call protocol ` takes care of recursion handling. be concatenated to the :exc:`RecursionError` message caused by the recursion depth limit. + .. seealso:: + The :c:func:`PyUnstable_ThreadState_SetStackProtection` function. + .. versionchanged:: 3.9 This function is now also available in the :ref:`limited API `. @@ -979,6 +1063,27 @@ these are the C equivalent to :func:`reprlib.recursive_repr`. Ends a :c:func:`Py_ReprEnter`. Must be called once for each invocation of :c:func:`Py_ReprEnter` that returns zero. +.. c:function:: int Py_GetRecursionLimit(void) + + Get the recursion limit for the current interpreter. It can be set with + :c:func:`Py_SetRecursionLimit`. The recursion limit prevents the + Python interpreter stack from growing infinitely. + + This function cannot fail, and the caller must hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`sys.getrecursionlimit` + +.. c:function:: void Py_SetRecursionLimit(int new_limit) + + Set the recursion limit for the current interpreter. + + This function cannot fail, and the caller must hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`sys.setrecursionlimit` .. _standardexceptions: @@ -1039,6 +1144,8 @@ Exception types * :exc:`FloatingPointError` * * .. c:var:: PyObject *PyExc_GeneratorExit * :exc:`GeneratorExit` + * * .. c:var:: PyObject *PyExc_ImportCycleError + * :exc:`ImportCycleError` * * .. c:var:: PyObject *PyExc_ImportError * :exc:`ImportError` * * .. c:var:: PyObject *PyExc_IndentationError @@ -1207,3 +1314,37 @@ Warning types .. versionadded:: 3.10 :c:data:`PyExc_EncodingWarning`. + + +Tracebacks +========== + +.. c:var:: PyTypeObject PyTraceBack_Type + + Type object for traceback objects. This is available as + :class:`types.TracebackType` in the Python layer. + + +.. c:function:: int PyTraceBack_Check(PyObject *op) + + Return true if *op* is a traceback object, false otherwise. This function + does not account for subtypes. + + +.. c:function:: int PyTraceBack_Here(PyFrameObject *f) + + Replace the :attr:`~BaseException.__traceback__` attribute on the current + exception with a new traceback prepending *f* to the existing chain. + + Calling this function without an exception set is undefined behavior. + + This function returns ``0`` on success, and returns ``-1`` with an + exception set on failure. + + +.. c:function:: int PyTraceBack_Print(PyObject *tb, PyObject *f) + + Write the traceback *tb* into the file *f*. + + This function returns ``0`` on success, and returns ``-1`` with an + exception set on failure. diff --git a/Doc/c-api/extension-modules.rst b/Doc/c-api/extension-modules.rst index 3d331e6ec12..7bc04970b19 100644 --- a/Doc/c-api/extension-modules.rst +++ b/Doc/c-api/extension-modules.rst @@ -8,7 +8,8 @@ Defining extension modules A C extension for CPython is a shared library (for example, a ``.so`` file on Linux, ``.pyd`` DLL on Windows), which is loadable into the Python process (for example, it is compiled with compatible compiler settings), and which -exports an :ref:`initialization function `. +exports an :dfn:`export hook` function (or an +old-style :ref:`initialization function `). To be importable by default (that is, by :py:class:`importlib.machinery.ExtensionFileLoader`), @@ -23,25 +24,127 @@ and must be named after the module name plus an extension listed in One suitable tool is Setuptools, whose documentation can be found at https://setuptools.pypa.io/en/latest/setuptools.html. -Normally, the initialization function returns a module definition initialized -using :c:func:`PyModuleDef_Init`. -This allows splitting the creation process into several phases: +.. _extension-export-hook: +Extension export hook +..................... + +.. versionadded:: 3.15 + + Support for the :samp:`PyModExport_{}` export hook was added in Python + 3.15. The older way of defining modules is still available: consult either + the :ref:`extension-pyinit` section or earlier versions of this + documentation if you plan to support earlier Python versions. + +The export hook must be an exported function with the following signature: + +.. c:function:: PyModuleDef_Slot *PyModExport_modulename(void) + +For modules with ASCII-only names, the :ref:`export hook ` +must be named :samp:`PyModExport_{}`, +with ```` replaced by the module's name. + +For non-ASCII module names, the export hook must instead be named +:samp:`PyModExportU_{}` (note the ``U``), with ```` encoded using +Python's *punycode* encoding with hyphens replaced by underscores. In Python: + +.. code-block:: python + + def hook_name(name): + try: + suffix = b'_' + name.encode('ascii') + except UnicodeEncodeError: + suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') + return b'PyModExport' + suffix + +The export hook returns an array of :c:type:`PyModuleDef_Slot` entries, +terminated by an entry with a slot ID of ``0``. +These slots describe how the module should be created and initialized. + +This array must remain valid and constant until interpreter shutdown. +Typically, it should use ``static`` storage. +Prefer using the :c:macro:`Py_mod_create` and :c:macro:`Py_mod_exec` slots +for any dynamic behavior. + +The export hook may return ``NULL`` with an exception set to signal failure. + +It is recommended to define the export hook function using a helper macro: + +.. c:macro:: PyMODEXPORT_FUNC + + Declare an extension module export hook. + This macro: + + * specifies the :c:expr:`PyModuleDef_Slot*` return type, + * adds any special linkage declarations required by the platform, and + * for C++, declares the function as ``extern "C"``. + +For example, a module called ``spam`` would be defined like this:: + + PyABIInfo_VAR(abi_info); + + static PyModuleDef_Slot spam_slots[] = { + {Py_mod_abi, &abi_info}, + {Py_mod_name, "spam"}, + {Py_mod_init, spam_init_function}, + ... + {0, NULL}, + }; + + PyMODEXPORT_FUNC + PyModExport_spam(void) + { + return spam_slots; + } + +The export hook is typically the only non-\ ``static`` +item defined in the module's C source. + +The hook should be kept short -- ideally, one line as above. +If you do need to use Python C API in this function, it is recommended to call +``PyABIInfo_Check(&abi_info, "modulename")`` first to raise an exception, +rather than crash, in common cases of ABI mismatch. + + +.. note:: + + It is possible to export multiple modules from a single shared library by + defining multiple export hooks. + However, importing them requires a custom importer or suitably named + copies/links of the extension file, because Python's import machinery only + finds the function corresponding to the filename. + See the `Multiple modules in one library `__ + section in :pep:`489` for details. + + +.. _multi-phase-initialization: + +Multi-phase initialization +.......................... + +The process of creating an extension module follows several phases: + +- Python finds and calls the export hook to get information on how to + create the module. - Before any substantial code is executed, Python can determine which capabilities the module supports, and it can adjust the environment or refuse loading an incompatible extension. -- By default, Python itself creates the module object -- that is, it does - the equivalent of :py:meth:`object.__new__` for classes. - It also sets initial attributes like :attr:`~module.__package__` and - :attr:`~module.__loader__`. -- Afterwards, the module object is initialized using extension-specific - code -- the equivalent of :py:meth:`~object.__init__` on classes. + Slots like :c:data:`Py_mod_abi`, :c:data:`Py_mod_gil` and + :c:data:`Py_mod_multiple_interpreters` influence this step. +- By default, Python itself then creates the module object -- that is, it does + the equivalent of calling :py:meth:`~object.__new__` when creating an object. + This step can be overridden using the :c:data:`Py_mod_create` slot. +- Python sets initial module attributes like :attr:`~module.__package__` and + :attr:`~module.__loader__`, and inserts the module object into + :py:attr:`sys.modules`. +- Afterwards, the module object is initialized in an extension-specific way + -- the equivalent of :py:meth:`~object.__init__` when creating an object, + or of executing top-level code in a Python-language module. + The behavior is specified using the :c:data:`Py_mod_exec` slot. This is called *multi-phase initialization* to distinguish it from the legacy -(but still supported) *single-phase initialization* scheme, -where the initialization function returns a fully constructed module. -See the :ref:`single-phase-initialization section below ` -for details. +(but still supported) :ref:`single-phase initialization `, +where an initialization function returns a fully constructed module. .. versionchanged:: 3.5 @@ -53,7 +156,7 @@ Multiple module instances By default, extension modules are not singletons. For example, if the :py:attr:`sys.modules` entry is removed and the module -is re-imported, a new module object is created, and typically populated with +is re-imported, a new module object is created and, typically, populated with fresh method and type objects. The old module is subject to normal garbage collection. This mirrors the behavior of pure-Python modules. @@ -83,36 +186,34 @@ A module may also be limited to the main interpreter using the :c:data:`Py_mod_multiple_interpreters` slot. -.. _extension-export-hook: +.. _extension-pyinit: -Initialization function -....................... +``PyInit`` function +................... -The initialization function defined by an extension module has the -following signature: +.. soft-deprecated:: 3.15 + + This functionality will not get new features, + but there are no plans to remove it. + +Instead of :c:func:`PyModExport_modulename`, an extension module can define +an older-style :dfn:`initialization function` with the signature: .. c:function:: PyObject* PyInit_modulename(void) Its name should be :samp:`PyInit_{}`, with ```` replaced by the name of the module. +For non-ASCII module names, use :samp:`PyInitU_{}` instead, with +```` encoded in the same way as for the +:ref:`export hook ` (that is, using Punycode +with underscores). -For modules with ASCII-only names, the function must instead be named -:samp:`PyInit_{}`, with ```` replaced by the name of the module. -When using :ref:`multi-phase-initialization`, non-ASCII module names -are allowed. In this case, the initialization function name is -:samp:`PyInitU_{}`, with ```` encoded using Python's -*punycode* encoding with hyphens replaced by underscores. In Python: +If a module exports both :samp:`PyInit_{}` and +:samp:`PyModExport_{}`, the :samp:`PyInit_{}` function +is ignored. -.. code-block:: python - - def initfunc_name(name): - try: - suffix = b'_' + name.encode('ascii') - except UnicodeEncodeError: - suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') - return b'PyInit' + suffix - -It is recommended to define the initialization function using a helper macro: +Like with :c:macro:`PyMODEXPORT_FUNC`, it is recommended to define the +initialization function using a helper macro: .. c:macro:: PyMODINIT_FUNC @@ -123,6 +224,34 @@ It is recommended to define the initialization function using a helper macro: * adds any special linkage declarations required by the platform, and * for C++, declares the function as ``extern "C"``. + +Normally, the initialization function (``PyInit_modulename``) returns +a :c:type:`PyModuleDef` instance with non-``NULL`` +:c:member:`~PyModuleDef.m_slots`. This allows Python to use +:ref:`multi-phase initialization `. + +Before it is returned, the ``PyModuleDef`` instance must be initialized +using the following function: + +.. c:function:: PyObject* PyModuleDef_Init(PyModuleDef *def) + + Ensure a module definition is a properly initialized Python object that + correctly reports its type and a reference count. + + Return *def* cast to ``PyObject*``, or ``NULL`` if an error occurred. + + Calling this function is required before returning a :c:type:`PyModuleDef` + from a module initialization function. + It should not be used in other contexts. + + Note that Python assumes that ``PyModuleDef`` structures are statically + allocated. + This function may return either a new reference or a borrowed one; + this reference must not be released. + + .. versionadded:: 3.5 + + For example, a module called ``spam`` would be defined like this:: static struct PyModuleDef spam_module = { @@ -137,59 +266,22 @@ For example, a module called ``spam`` would be defined like this:: return PyModuleDef_Init(&spam_module); } -It is possible to export multiple modules from a single shared library by -defining multiple initialization functions. However, importing them requires -using symbolic links or a custom importer, because by default only the -function corresponding to the filename is found. -See the `Multiple modules in one library `__ -section in :pep:`489` for details. - -The initialization function is typically the only non-\ ``static`` -item defined in the module's C source. - - -.. _multi-phase-initialization: - -Multi-phase initialization -.......................... - -Normally, the :ref:`initialization function ` -(``PyInit_modulename``) returns a :c:type:`PyModuleDef` instance with -non-``NULL`` :c:member:`~PyModuleDef.m_slots`. -Before it is returned, the ``PyModuleDef`` instance must be initialized -using the following function: - - -.. c:function:: PyObject* PyModuleDef_Init(PyModuleDef *def) - - Ensure a module definition is a properly initialized Python object that - correctly reports its type and a reference count. - - Return *def* cast to ``PyObject*``, or ``NULL`` if an error occurred. - - Calling this function is required for :ref:`multi-phase-initialization`. - It should not be used in other contexts. - - Note that Python assumes that ``PyModuleDef`` structures are statically - allocated. - This function may return either a new reference or a borrowed one; - this reference must not be released. - - .. versionadded:: 3.5 - .. _single-phase-initialization: Legacy single-phase initialization -.................................. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. soft-deprecated:: 3.15 -.. attention:: Single-phase initialization is a legacy mechanism to initialize extension modules, with known drawbacks and design flaws. Extension module authors are encouraged to use multi-phase initialization instead. -In single-phase initialization, the -:ref:`initialization function ` (``PyInit_modulename``) + However, there are no plans to remove support for it. + +In single-phase initialization, the old-style +:ref:`initialization function ` (``PyInit_modulename``) should create, populate and return a module object. This is typically done using :c:func:`PyModule_Create` and functions like :c:func:`PyModule_AddObjectRef`. @@ -242,6 +334,8 @@ in the following ways: * Single-phase modules support module lookup functions like :c:func:`PyState_FindModule`. +* The module's :c:member:`PyModuleDef.m_slots` must be NULL. + .. [#testsinglephase] ``_testsinglephase`` is an internal module used in CPython's self-test suite; your installation may or may not include it. diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index e9019a0d500..dcafefdc045 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -2,7 +2,7 @@ .. _fileobjects: -File Objects +File objects ------------ .. index:: pair: object; file @@ -93,6 +93,29 @@ the :mod:`io` APIs instead. .. versionadded:: 3.8 +.. c:function:: PyObject *PyFile_OpenCodeObject(PyObject *path) + + Open *path* with the mode ``'rb'``. *path* must be a Python :class:`str` + object. The behavior of this function may be overridden by + :c:func:`PyFile_SetOpenCodeHook` to allow for some preprocessing of the + text. + + This is analogous to :func:`io.open_code` in Python. + + On success, this function returns a :term:`strong reference` to a Python + file object. On failure, this function returns ``NULL`` with an exception + set. + + .. versionadded:: 3.8 + + +.. c:function:: PyObject *PyFile_OpenCode(const char *path) + + Similar to :c:func:`PyFile_OpenCodeObject`, but *path* is a + UTF-8 encoded :c:expr:`const char*`. + + .. versionadded:: 3.8 + .. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags) @@ -100,11 +123,34 @@ the :mod:`io` APIs instead. Write object *obj* to file object *p*. The only supported flag for *flags* is :c:macro:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written - instead of the :func:`repr`. Return ``0`` on success or ``-1`` on failure; the - appropriate exception will be set. + instead of the :func:`repr`. + If *obj* is ``NULL``, write the string ``""``. + + Return ``0`` on success or ``-1`` on failure; the + appropriate exception will be set. .. c:function:: int PyFile_WriteString(const char *s, PyObject *p) Write string *s* to file object *p*. Return ``0`` on success or ``-1`` on failure; the appropriate exception will be set. + + +Soft-deprecated API +^^^^^^^^^^^^^^^^^^^ + +.. soft-deprecated:: 3.15 + +These are APIs that were included in Python's C API +by mistake. They are documented solely for completeness; use other +``PyFile*`` APIs instead. + +.. c:function:: PyObject *PyFile_NewStdPrinter(int fd) + + Use :c:func:`PyFile_FromFd` with defaults (``fd, NULL, "w", -1, NULL, NULL, NULL, 0``) instead. + +.. c:var:: PyTypeObject PyStdPrinter_Type + + Type of file-like objects used internally at Python startup when :py:mod:`io` is + not yet available. + Use Python :py:func:`open` or :c:func:`PyFile_FromFd` to create file objects instead. diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 489676caa3a..a12ad11abb1 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -78,6 +78,109 @@ Floating-Point Objects Return the minimum normalized positive float *DBL_MIN* as C :c:expr:`double`. +.. c:macro:: Py_INFINITY + + This macro expands to a constant expression of type :c:expr:`double`, that + represents the positive infinity. + + It is equivalent to the :c:macro:`!INFINITY` macro from the C11 standard + ```` header. + + .. soft-deprecated:: 3.15 + + +.. c:macro:: Py_NAN + + This macro expands to a constant expression of type :c:expr:`double`, that + represents a quiet not-a-number (qNaN) value. + + On most platforms, this is equivalent to the :c:macro:`!NAN` macro from + the C11 standard ```` header. + + +.. c:macro:: Py_HUGE_VAL + + Equivalent to :c:macro:`!INFINITY`. + + .. soft-deprecated:: 3.14 + + +.. c:macro:: Py_MATH_E + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.e` constant. + + +.. c:macro:: Py_MATH_El + + High precision (long double) definition of :data:`~math.e` constant. + + .. deprecated-removed:: 3.15 3.20 + + +.. c:macro:: Py_MATH_PI + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.pi` constant. + + +.. c:macro:: Py_MATH_PIl + + High precision (long double) definition of :data:`~math.pi` constant. + + .. deprecated-removed:: 3.15 3.20 + + +.. c:macro:: Py_MATH_TAU + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.tau` constant. + + .. versionadded:: 3.6 + + +.. c:macro:: Py_RETURN_NAN + + Return :data:`math.nan` from a function. + + On most platforms, this is equivalent to ``return PyFloat_FromDouble(NAN)``. + + +.. c:macro:: Py_RETURN_INF(sign) + + Return :data:`math.inf` or :data:`-math.inf ` from a function, + depending on the sign of *sign*. + + On most platforms, this is equivalent to the following:: + + return PyFloat_FromDouble(copysign(INFINITY, sign)); + + +.. c:macro:: Py_IS_FINITE(X) + + Return ``1`` if the given floating-point number *X* is finite, + that is, it is normal, subnormal or zero, but not infinite or NaN. + Return ``0`` otherwise. + + .. soft-deprecated:: 3.14 + Use :c:macro:`!isfinite` instead. + + +.. c:macro:: Py_IS_INFINITY(X) + + Return ``1`` if the given floating-point number *X* is positive or negative + infinity. Return ``0`` otherwise. + + .. soft-deprecated:: 3.14 + Use :c:macro:`!isinf` instead. + + +.. c:macro:: Py_IS_NAN(X) + + Return ``1`` if the given floating-point number *X* is a not-a-number (NaN) + value. Return ``0`` otherwise. + + .. soft-deprecated:: 3.14 + Use :c:macro:`!isnan` instead. + + Pack and Unpack functions ------------------------- @@ -85,24 +188,23 @@ The pack and unpack functions provide an efficient platform-independent way to store floating-point values as byte strings. The Pack routines produce a bytes string from a C :c:expr:`double`, and the Unpack routines produce a C :c:expr:`double` from such a bytes string. The suffix (2, 4 or 8) specifies the -number of bytes in the bytes string. +number of bytes in the bytes string: -On platforms that appear to use IEEE 754 formats these functions work by -copying bits. On other platforms, the 2-byte format is identical to the IEEE -754 binary16 half-precision format, the 4-byte format (32-bit) is identical to -the IEEE 754 binary32 single precision format, and the 8-byte format to the -IEEE 754 binary64 double precision format, although the packing of INFs and -NaNs (if such things exist on the platform) isn't handled correctly, and -attempting to unpack a bytes string containing an IEEE INF or NaN will raise an -exception. +* The 2-byte format is the IEEE 754 binary16 half-precision format. +* The 4-byte format is the IEEE 754 binary32 single-precision format. +* The 8-byte format is the IEEE 754 binary64 double-precision format. -Note that NaNs type may not be preserved on IEEE platforms (silent NaN become -quiet), for example on x86 systems in 32-bit mode. +The NaN type may not be preserved on some platforms while unpacking (signaling +NaNs become quiet NaNs), for example on x86 systems in 32-bit mode. +It's assumed that the :c:expr:`double` type has the IEEE 754 binary64 double +precision format. What happens if it's not true is partly accidental (alas). On non-IEEE platforms with more precision, or larger dynamic range, than IEEE 754 supports, not all values can be packed; on non-IEEE platforms with less -precision, or smaller dynamic range, not all values can be unpacked. What -happens in such cases is partly accidental (alas). +precision, or smaller dynamic range, not all values can be unpacked. The +packing of special numbers like INFs and NaNs (if such things exist on the +platform) may not be handled correctly, and attempting to unpack a bytes string +containing an IEEE INF or NaN may raise an exception. .. versionadded:: 3.11 @@ -111,19 +213,14 @@ Pack functions The pack routines write 2, 4 or 8 bytes, starting at *p*. *le* is an :c:expr:`int` argument, non-zero if you want the bytes string in little-endian -format (exponent last, at ``p+1``, ``p+3``, or ``p+6`` ``p+7``), zero if you -want big-endian format (exponent first, at *p*). The :c:macro:`PY_BIG_ENDIAN` -constant can be used to use the native endian: it is equal to ``1`` on big -endian processor, or ``0`` on little endian processor. +format (exponent last, at ``p+1``, ``p+3``, or ``p+6`` and ``p+7``), zero if you +want big-endian format (exponent first, at *p*). Use the :c:macro:`!PY_LITTLE_ENDIAN` +constant to select the native endian: it is equal to ``0`` on big +endian processor, or ``1`` on little endian processor. Return value: ``0`` if all is OK, ``-1`` if error (and an exception is set, most likely :exc:`OverflowError`). -There are two problems on non-IEEE platforms: - -* What this does is undefined if *x* is a NaN or infinity. -* ``-0.0`` and ``+0.0`` produce the same bytes string. - .. c:function:: int PyFloat_Pack2(double x, char *p, int le) Pack a C double as the IEEE 754 binary16 half-precision format. @@ -136,6 +233,9 @@ There are two problems on non-IEEE platforms: Pack a C double as the IEEE 754 binary64 double precision format. + .. impl-detail:: + This function always succeeds in CPython. + Unpack functions ^^^^^^^^^^^^^^^^ @@ -143,16 +243,16 @@ Unpack functions The unpack routines read 2, 4 or 8 bytes, starting at *p*. *le* is an :c:expr:`int` argument, non-zero if the bytes string is in little-endian format (exponent last, at ``p+1``, ``p+3`` or ``p+6`` and ``p+7``), zero if big-endian -(exponent first, at *p*). The :c:macro:`PY_BIG_ENDIAN` constant can be used to -use the native endian: it is equal to ``1`` on big endian processor, or ``0`` +(exponent first, at *p*). Use the :c:macro:`!PY_LITTLE_ENDIAN` constant to +select the native endian: it is equal to ``0`` on big endian processor, or ``1`` on little endian processor. Return value: The unpacked double. On error, this is ``-1.0`` and :c:func:`PyErr_Occurred` is true (and an exception is set, most likely :exc:`OverflowError`). -Note that on a non-IEEE platform this will refuse to unpack a bytes string that -represents a NaN or infinity. +.. impl-detail:: + These functions always succeed in CPython. .. c:function:: double PyFloat_Unpack2(const char *p, int le) diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 1a52e146a69..4159ff6e596 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -1,6 +1,6 @@ .. highlight:: c -Frame Objects +Frame objects ------------- .. c:type:: PyFrameObject @@ -29,6 +29,12 @@ See also :ref:`Reflection `. Previously, this type was only available after including ````. +.. c:function:: PyFrameObject *PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals, PyObject *locals) + + Create a new frame object. This function returns a :term:`strong reference` + to the new frame object on success, and returns ``NULL`` with an exception + set on failure. + .. c:function:: int PyFrame_Check(PyObject *obj) Return non-zero if *obj* is a frame object. @@ -44,6 +50,7 @@ See also :ref:`Reflection `. Return a :term:`strong reference`, or ``NULL`` if *frame* has no outer frame. + This raises no exceptions. .. versionadded:: 3.9 @@ -140,7 +147,7 @@ See also :ref:`Reflection `. Return the line number that *frame* is currently executing. -Frame Locals Proxies +Frame locals proxies ^^^^^^^^^^^^^^^^^^^^ .. versionadded:: 3.13 @@ -161,7 +168,52 @@ See :pep:`667` for more information. Return non-zero if *obj* is a frame :func:`locals` proxy. -Internal Frames + +Legacy local variable APIs +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These APIs are :term:`soft deprecated`. As of Python 3.13, they do nothing. +They exist solely for backwards compatibility. + + +.. c:function:: void PyFrame_LocalsToFast(PyFrameObject *f, int clear) + + Prior to Python 3.13, this function would copy the :attr:`~frame.f_locals` + attribute of *f* to the internal "fast" array of local variables, allowing + changes in frame objects to be visible to the interpreter. If *clear* was + true, this function would process variables that were unset in the locals + dictionary. + + .. soft-deprecated:: 3.13 + This function now does nothing. + + +.. c:function:: void PyFrame_FastToLocals(PyFrameObject *f) + + Prior to Python 3.13, this function would copy the internal "fast" array + of local variables (which is used by the interpreter) to the + :attr:`~frame.f_locals` attribute of *f*, allowing changes in local + variables to be visible to frame objects. + + .. soft-deprecated:: 3.13 + This function now does nothing. + + +.. c:function:: int PyFrame_FastToLocalsWithError(PyFrameObject *f) + + Prior to Python 3.13, this function was similar to + :c:func:`PyFrame_FastToLocals`, but would return ``0`` on success, and + ``-1`` with an exception set on failure. + + .. soft-deprecated:: 3.13 + This function now does nothing. + + +.. seealso:: + :pep:`667` + + +Internal frames ^^^^^^^^^^^^^^^ Unless using :pep:`523`, you will not need this. @@ -191,5 +243,3 @@ Unless using :pep:`523`, you will not need this. Return the currently executing line number, or -1 if there is no line number. .. versionadded:: 3.12 - - diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 5fb8567ef8c..609b5e885b6 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -102,6 +102,15 @@ There are a few functions specific to Python functions. dictionary of arguments or ``NULL``. +.. c:function:: int PyFunction_SetKwDefaults(PyObject *op, PyObject *defaults) + + Set the keyword-only argument default values of the function object *op*. + *defaults* must be a dictionary of keyword-only arguments or ``Py_None``. + + This function returns ``0`` on success, and returns ``-1`` with an exception + set on failure. + + .. c:function:: PyObject* PyFunction_GetClosure(PyObject *op) Return the closure associated with the function object *op*. This can be ``NULL`` @@ -175,6 +184,9 @@ There are a few functions specific to Python functions. .. versionadded:: 3.12 + - ``PyFunction_PYFUNC_EVENT_MODIFY_QUALNAME`` + + .. versionadded:: 3.15 .. c:type:: int (*PyFunction_WatchCallback)(PyFunction_WatchEvent event, PyFunctionObject *func, PyObject *new_value) @@ -197,7 +209,7 @@ There are a few functions specific to Python functions. runtime behavior depending on optimization decisions, it does not change the semantics of the Python code being executed. - If *event* is ``PyFunction_EVENT_DESTROY``, Taking a reference in the + If *event* is ``PyFunction_EVENT_DESTROY``, taking a reference in the callback to the about-to-be-destroyed function will resurrect it, preventing it from being freed at this time. When the resurrected object is destroyed later, any watcher callbacks active at that time will be called again. diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index f6fa52b36c5..9c9c97f7b85 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -220,38 +220,6 @@ The :c:member:`~PyTypeObject.tp_traverse` handler accepts a function parameter o detection; it's not expected that users will need to write their own visitor functions. -The :c:member:`~PyTypeObject.tp_traverse` handler must have the following type: - - -.. c:type:: int (*traverseproc)(PyObject *self, visitproc visit, void *arg) - - Traversal function for a container object. Implementations must call the - *visit* function for each object directly contained by *self*, with the - parameters to *visit* being the contained object and the *arg* value passed - to the handler. The *visit* function must not be called with a ``NULL`` - object argument. If *visit* returns a non-zero value that value should be - returned immediately. - -To simplify writing :c:member:`~PyTypeObject.tp_traverse` handlers, a :c:func:`Py_VISIT` macro is -provided. In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse` implementation -must name its arguments exactly *visit* and *arg*: - - -.. c:macro:: Py_VISIT(o) - - If the :c:expr:`PyObject *` *o* is not ``NULL``, call the *visit* callback, with arguments *o* - and *arg*. If *visit* returns a non-zero value, then return it. - Using this macro, :c:member:`~PyTypeObject.tp_traverse` handlers - look like:: - - static int - my_traverse(Noddy *self, visitproc visit, void *arg) - { - Py_VISIT(self->foo); - Py_VISIT(self->bar); - return 0; - } - The :c:member:`~PyTypeObject.tp_clear` handler must be of the :c:type:`inquiry` type, or ``NULL`` if the object is immutable. @@ -266,6 +234,225 @@ if the object is immutable. in a reference cycle. +.. _gc-traversal: + +Traversal +--------- + +The :c:member:`~PyTypeObject.tp_traverse` handler must have the following type: + +.. c:type:: int (*traverseproc)(PyObject *self, visitproc visit, void *arg) + + Traversal function for a garbage-collected object, used by the garbage + collector to detect reference cycles. + Implementations must call the + *visit* function for each object directly contained by *self*, with the + parameters to *visit* being the contained object and the *arg* value passed + to the handler. The *visit* function must not be called with a ``NULL`` + object argument. If *visit* returns a non-zero value, that value should be + returned immediately. + + A typical :c:member:`!tp_traverse` function calls the :c:func:`Py_VISIT` + convenience macro on each of the instance's members that are Python + objects that the instance owns. + For example, this is a (slightly outdated) traversal function for + the :py:class:`threading.local` class:: + + static int + local_traverse(PyObject *op, visitproc visit, void *arg) + { + localobject *self = (localobject *) op; + Py_VISIT(Py_TYPE(self)); + Py_VISIT(self->args); + Py_VISIT(self->kw); + Py_VISIT(self->dict); + return 0; + } + + .. note:: + :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to + :c:func:`!local_traverse` to have these specific names; don't name them just + anything. + + Instances of :ref:`heap-allocated types ` hold a reference to + their type. Their traversal function must therefore visit the type:: + + Py_VISIT(Py_TYPE(self)); + + Alternately, the type may delegate this responsibility by + calling ``tp_traverse`` of a heap-allocated superclass (or another + heap-allocated type, if applicable). + If they do not, the type object may not be garbage-collected. + + If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the + :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call + :c:func:`PyObject_VisitManagedDict` like this:: + + int err = PyObject_VisitManagedDict((PyObject*)self, visit, arg); + if (err) { + return err; + } + + Only the members that the instance *owns* (by having + :term:`strong references ` to them) must be + visited. For instance, if an object supports weak references via the + :c:member:`~PyTypeObject.tp_weaklist` slot, the pointer supporting + the linked list (what *tp_weaklist* points to) must **not** be + visited as the instance does not directly own the weak references to itself. + + The traversal function has a limitation: + + .. warning:: + + The traversal function must not have any side effects. Implementations + may not modify the reference counts of any Python objects nor create or + destroy any Python objects, directly or indirectly. + + This means that *most* Python C API functions may not be used, since + they can raise a new exception, return a new reference to a result object, + have internal logic that uses side effects. + Also, unless documented otherwise, functions that happen to not have side + effects may start having them in future versions, without warning. + + For a list of safe functions, see a + :ref:`separate section ` below. + + .. note:: + + The :c:func:`Py_VISIT` call may be skipped for those members that provably + cannot participate in reference cycles. + In the ``local_traverse`` example above, there is also a ``self->key`` + member, but it can only be ``NULL`` or a Python string and therefore + cannot be part of a reference cycle. + + On the other hand, even if you know a member can never be part of a cycle, + as a debugging aid you may want to visit it anyway just so the :mod:`gc` + module's :func:`~gc.get_referents` function will include it. + + .. note:: + + The :c:member:`~PyTypeObject.tp_traverse` function can be called from any + thread. + + .. impl-detail:: + + Garbage collection is a "stop-the-world" operation: + even in :term:`free threading` builds, only one thread state is + :term:`attached ` when :c:member:`!tp_traverse` + handlers run. + + .. versionchanged:: 3.9 + + Heap-allocated types are expected to visit ``Py_TYPE(self)`` in + ``tp_traverse``. In earlier versions of Python, due to + `bug 40217 `_, doing this + may lead to crashes in subclasses. + +To simplify writing :c:member:`~PyTypeObject.tp_traverse` handlers, +a :c:func:`Py_VISIT` macro is provided. +In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse` +implementation must name its arguments exactly *visit* and *arg*: + +.. c:macro:: Py_VISIT(o) + + If the :c:expr:`PyObject *` *o* is not ``NULL``, call the *visit* + callback, with arguments *o* and *arg*. + If *visit* returns a non-zero value, then return it. + + This corresponds roughly to:: + + #define Py_VISIT(o) \ + if (op) { \ + int visit_result = visit(o, arg); \ + if (visit_result != 0) { \ + return visit_result; \ + } \ + } + + +Traversal-safe functions +^^^^^^^^^^^^^^^^^^^^^^^^ + +The following functions and macros are safe to use in a +:c:member:`~PyTypeObject.tp_traverse` handler: + +* the *visit* function passed to ``tp_traverse`` +* :c:func:`Py_VISIT` +* :c:func:`Py_SIZE` +* :c:func:`Py_TYPE`: if called from a :c:member:`!tp_traverse` handler, + :c:func:`!Py_TYPE`'s result will be valid for the duration of the handler call +* :c:func:`PyObject_VisitManagedDict` +* :c:func:`PyObject_TypeCheck`, :c:func:`PyType_IsSubtype`, + :c:func:`PyType_HasFeature` +* :samp:`Py{}_Check` and :samp:`Py{}_CheckExact` -- for example, + :c:func:`PyTuple_Check` +* :ref:`duringgc-functions` + +.. _duringgc-functions: + +"DuringGC" functions +^^^^^^^^^^^^^^^^^^^^ + +The following functions should *only* be used in a +:c:member:`~PyTypeObject.tp_traverse` handler; calling them in other +contexts may have unintended consequences. + +These functions act like their counterparts without the ``_DuringGC`` suffix, +but they are guaranteed to not have side effects, they do not set an exception +on failure, and they return/set :term:`borrowed references ` +as detailed in the individual documentation. + +Note that these functions may fail (return ``NULL`` or ``-1``), +but as they do not set an exception, no error information is available. +In some cases, failure is not distinguishable from a successful ``NULL`` result. + +.. c:function:: void *PyObject_GetTypeData_DuringGC(PyObject *o, PyTypeObject *cls) + void *PyObject_GetItemData_DuringGC(PyObject *o) + void *PyType_GetModuleState_DuringGC(PyTypeObject *type) + void *PyModule_GetState_DuringGC(PyObject *module) + int PyModule_GetToken_DuringGC(PyObject *module, void** result) + + See :ref:`duringgc-functions` for common information. + + .. versionadded:: next + + .. seealso:: + + :c:func:`PyObject_GetTypeData`, + :c:func:`PyObject_GetItemData`, + :c:func:`PyType_GetModuleState`, + :c:func:`PyModule_GetState`, + :c:func:`PyModule_GetToken`, + :c:func:`PyType_GetBaseByToken` + +.. c:function:: int PyType_GetBaseByToken_DuringGC(PyTypeObject *type, void *tp_token, PyTypeObject **result) + + See :ref:`duringgc-functions` for common information. + + Sets *\*result* to a :term:`borrowed reference` rather than a strong one. + The reference is valid for the duration + of the :c:member:`!tp_traverse` handler call. + + .. versionadded:: next + + .. seealso:: :c:func:`PyType_GetBaseByToken` + +.. c:function:: PyObject* PyType_GetModule_DuringGC(PyTypeObject *type) + PyObject* PyType_GetModuleByToken_DuringGC(PyTypeObject *type, const void *mod_token) + + See :ref:`duringgc-functions` for common information. + + These functions return a :term:`borrowed reference`, which is + valid for the duration of the :c:member:`!tp_traverse` handler call. + + .. versionadded:: next + + .. seealso:: + + :c:func:`PyType_GetModule`, + :c:func:`PyType_GetModuleByToken` + + Controlling the Garbage Collector State --------------------------------------- diff --git a/Doc/c-api/gen.rst b/Doc/c-api/gen.rst index 0eb5922f6da..ed121726b89 100644 --- a/Doc/c-api/gen.rst +++ b/Doc/c-api/gen.rst @@ -44,3 +44,55 @@ than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`. with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. A reference to *frame* is stolen by this function. The *frame* argument must not be ``NULL``. + + +.. c:function:: PyCodeObject* PyGen_GetCode(PyGenObject *gen) + + Return a new :term:`strong reference` to the code object wrapped by *gen*. + This function always succeeds. + + +Asynchronous Generator Objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. seealso:: + :pep:`525` + +.. c:var:: PyTypeObject PyAsyncGen_Type + + The type object corresponding to asynchronous generator objects. This is + available as :class:`types.AsyncGeneratorType` in the Python layer. + + .. versionadded:: 3.6 + +.. c:function:: PyObject *PyAsyncGen_New(PyFrameObject *frame, PyObject *name, PyObject *qualname) + + Create a new asynchronous generator wrapping *frame*, with ``__name__`` and + ``__qualname__`` set to *name* and *qualname*. *frame* is stolen by this + function and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to the + new asynchronous generator. On failure, this function returns ``NULL`` + with an exception set. + + .. versionadded:: 3.6 + +.. c:function:: int PyAsyncGen_CheckExact(PyObject *op) + + Return true if *op* is an asynchronous generator object, false otherwise. + This function always succeeds. + + .. versionadded:: 3.6 + + +Deprecated API +^^^^^^^^^^^^^^ + +.. c:macro:: PyAsyncGenASend_CheckExact(op) + + This is an API that was included in Python's C API + by mistake. + + It is solely here for completeness; do not use this API. + + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst index b5fe93573a1..1ad712b0ce4 100644 --- a/Doc/c-api/hash.rst +++ b/Doc/c-api/hash.rst @@ -11,42 +11,98 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. .. versionadded:: 3.2 + .. c:type:: Py_uhash_t Hash value type: unsigned integer. .. versionadded:: 3.2 + +.. c:macro:: Py_HASH_ALGORITHM + + A numerical value indicating the algorithm for hashing of :class:`str`, + :class:`bytes`, and :class:`memoryview`. + + The algorithm name is exposed by :data:`sys.hash_info.algorithm`. + + .. versionadded:: 3.4 + + +.. c:macro:: Py_HASH_FNV + Py_HASH_SIPHASH24 + Py_HASH_SIPHASH13 + + Numerical values to compare to :c:macro:`Py_HASH_ALGORITHM` to determine + which algorithm is used for hashing. The hash algorithm can be configured + via the configure :option:`--with-hash-algorithm` option. + + .. versionadded:: 3.4 + Add :c:macro:`!Py_HASH_FNV` and :c:macro:`!Py_HASH_SIPHASH24`. + + .. versionadded:: 3.11 + Add :c:macro:`!Py_HASH_SIPHASH13`. + + +.. c:macro:: Py_HASH_CUTOFF + + Buffers of length in range ``[1, Py_HASH_CUTOFF)`` are hashed using DJBX33A + instead of the algorithm described by :c:macro:`Py_HASH_ALGORITHM`. + + - A :c:macro:`!Py_HASH_CUTOFF` of 0 disables the optimization. + - :c:macro:`!Py_HASH_CUTOFF` must be non-negative and less or equal than 7. + + 32-bit platforms should use a cutoff smaller than 64-bit platforms because + it is easier to create colliding strings. A cutoff of 7 on 64-bit platforms + and 5 on 32-bit platforms should provide a decent safety margin. + + This corresponds to the :data:`sys.hash_info.cutoff` constant. + + .. versionadded:: 3.4 + + .. c:macro:: PyHASH_MODULUS - The `Mersenne prime `_ ``P = 2**n -1``, used for numeric hash scheme. + The `Mersenne prime `_ ``P = 2**n -1``, + used for numeric hash scheme. + + This corresponds to the :data:`sys.hash_info.modulus` constant. .. versionadded:: 3.13 + .. c:macro:: PyHASH_BITS The exponent ``n`` of ``P`` in :c:macro:`PyHASH_MODULUS`. .. versionadded:: 3.13 + .. c:macro:: PyHASH_MULTIPLIER Prime multiplier used in string and various other hashes. .. versionadded:: 3.13 + .. c:macro:: PyHASH_INF The hash value returned for a positive infinity. + This corresponds to the :data:`sys.hash_info.inf` constant. + .. versionadded:: 3.13 + .. c:macro:: PyHASH_IMAG The multiplier used for the imaginary part of a complex number. + This corresponds to the :data:`sys.hash_info.imag` constant. + .. versionadded:: 3.13 + .. c:type:: PyHash_FuncDef Hash function definition used by :c:func:`PyHash_GetFuncDef`. @@ -59,14 +115,20 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. Hash function name (UTF-8 encoded string). + This corresponds to the :data:`sys.hash_info.algorithm` constant. + .. c:member:: const int hash_bits Internal size of the hash value in bits. + This corresponds to the :data:`sys.hash_info.hash_bits` constant. + .. c:member:: const int seed_bits Size of seed input in bits. + This corresponds to the :data:`sys.hash_info.seed_bits` constant. + .. versionadded:: 3.4 diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 8eabc0406b1..e2d363b911a 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -129,8 +129,7 @@ Importing Modules of :class:`~importlib.machinery.SourceFileLoader` otherwise. The module's :attr:`~module.__file__` attribute will be set to the code - object's :attr:`~codeobject.co_filename`. If applicable, - :attr:`~module.__cached__` will also be set. + object's :attr:`~codeobject.co_filename`. This function will reload the module if it was already imported. See :c:func:`PyImport_ReloadModule` for the intended way to reload a module. @@ -142,10 +141,13 @@ Importing Modules :c:func:`PyImport_ExecCodeModuleWithPathnames`. .. versionchanged:: 3.12 - The setting of :attr:`~module.__cached__` and :attr:`~module.__loader__` + The setting of ``__cached__`` and :attr:`~module.__loader__` is deprecated. See :class:`~importlib.machinery.ModuleSpec` for alternatives. + .. versionchanged:: 3.15 + ``__cached__`` is no longer set. + .. c:function:: PyObject* PyImport_ExecCodeModuleEx(const char *name, PyObject *co, const char *pathname) @@ -157,16 +159,19 @@ Importing Modules .. c:function:: PyObject* PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, PyObject *cpathname) - Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`~module.__cached__` - attribute of the module object is set to *cpathname* if it is - non-``NULL``. Of the three functions, this is the preferred one to use. + Like :c:func:`PyImport_ExecCodeModuleEx`, but the path to any compiled file + via *cpathname* is used appropriately when non-``NULL``. Of the three + functions, this is the preferred one to use. .. versionadded:: 3.3 .. versionchanged:: 3.12 - Setting :attr:`~module.__cached__` is deprecated. See + Setting ``__cached__`` is deprecated. See :class:`~importlib.machinery.ModuleSpec` for alternatives. + .. versionchanged:: 3.15 + ``__cached__`` no longer set. + .. c:function:: PyObject* PyImport_ExecCodeModuleWithPathnames(const char *name, PyObject *co, const char *pathname, const char *cpathname) @@ -314,6 +319,13 @@ Importing Modules initialization. +.. c:var:: struct _inittab *PyImport_Inittab + + The table of built-in modules used by Python initialization. Do not use this directly; + use :c:func:`PyImport_AppendInittab` and :c:func:`PyImport_ExtendInittab` + instead. + + .. c:function:: PyObject* PyImport_ImportModuleAttr(PyObject *mod_name, PyObject *attr_name) Import the module *mod_name* and get its attribute *attr_name*. @@ -333,3 +345,78 @@ Importing Modules strings instead of Python :class:`str` objects. .. versionadded:: 3.14 + +.. c:function:: PyImport_LazyImportsMode PyImport_GetLazyImportsMode() + + Gets the current lazy imports mode. + + .. versionadded:: 3.15 + +.. c:function:: PyObject* PyImport_GetLazyImportsFilter() + + Return a :term:`strong reference` to the current lazy imports filter, + or ``NULL`` if none exists. This function always succeeds. + + .. versionadded:: 3.15 + +.. c:function:: int PyImport_SetLazyImportsMode(PyImport_LazyImportsMode mode) + + Similar to :c:func:`PyImport_ImportModuleAttr`, but names are UTF-8 encoded + strings instead of Python :class:`str` objects. + + This function always returns ``0``. + + .. versionadded:: 3.15 + +.. c:function:: int PyImport_SetLazyImportsFilter(PyObject *filter) + + Sets the current lazy imports filter. The *filter* should be a callable that + will receive ``(importing_module_name, imported_module_name, [fromlist])`` + when an import can potentially be lazy. The ``imported_module_name`` value + is the resolved module name, so ``lazy from .spam import eggs`` passes + ``package.spam``. The callable must return ``True`` if the import should be + lazy and ``False`` otherwise. + + Return ``0`` on success and ``-1`` with an exception set otherwise. + + .. versionadded:: 3.15 + +.. c:type:: PyImport_LazyImportsMode + + Enumeration of possible lazy import modes. + + .. c:enumerator:: PyImport_LAZY_NORMAL + + Respect the ``lazy`` keyword in source code. This is the default mode. + + .. c:enumerator:: PyImport_LAZY_ALL + + Make all imports lazy by default. + + .. c:enumerator:: PyImport_LAZY_NONE + + Disable lazy imports entirely. Even explicit ``lazy`` statements become + eager imports. + + .. versionadded:: 3.15 + +.. c:function:: PyObject* PyImport_CreateModuleFromInitfunc(PyObject *spec, PyObject* (*initfunc)(void)) + + This function is a building block that enables embedders to implement + the :py:meth:`~importlib.abc.Loader.create_module` step of custom + static extension importers (e.g. of statically-linked extensions). + + *spec* must be a :class:`~importlib.machinery.ModuleSpec` object. + + *initfunc* must be an :ref:`initialization function `, + the same as for :c:func:`PyImport_AppendInittab`. + + On success, create and return a module object. + This module will not be initialized; call :c:func:`PyModule_Exec` + to initialize it. + (Custom importers should do this in their + :py:meth:`~importlib.abc.Loader.exec_module` method.) + + On error, return NULL with an exception set. + + .. versionadded:: 3.15 diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst index e9df2a304d9..eabe00f4004 100644 --- a/Doc/c-api/index.rst +++ b/Doc/c-api/index.rst @@ -1,7 +1,7 @@ .. _c-api-index: ################################## - Python/C API Reference Manual + Python/C API reference manual ################################## This manual documents the API used by C and C++ programmers who want to write @@ -21,7 +21,12 @@ document the API functions in detail. utilities.rst abstract.rst concrete.rst - init.rst + interp-lifecycle.rst + threads.rst + synchronization.rst + tls.rst + subinterpreters.rst + profiling.rst init_config.rst memory.rst objimpl.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index bb9e08acee1..e56c67f9534 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1,2450 +1,13 @@ -.. highlight:: c +:orphan: +Initialization, finalization, and threads +========================================= -.. _initialization: +This page has been split up into the following: -***************************************** -Initialization, Finalization, and Threads -***************************************** - -See :ref:`Python Initialization Configuration ` for details -on how to configure the interpreter prior to initialization. - -.. _pre-init-safe: - -Before Python Initialization -============================ - -In an application embedding Python, the :c:func:`Py_Initialize` function must -be called before using any other Python/C API functions; with the exception of -a few functions and the :ref:`global configuration variables -`. - -The following functions can be safely called before Python is initialized: - -* Functions that initialize the interpreter: - - * :c:func:`Py_Initialize` - * :c:func:`Py_InitializeEx` - * :c:func:`Py_InitializeFromConfig` - * :c:func:`Py_BytesMain` - * :c:func:`Py_Main` - * the runtime pre-initialization functions covered in :ref:`init-config` - -* Configuration functions: - - * :c:func:`PyImport_AppendInittab` - * :c:func:`PyImport_ExtendInittab` - * :c:func:`!PyInitFrozenExtensions` - * :c:func:`PyMem_SetAllocator` - * :c:func:`PyMem_SetupDebugHooks` - * :c:func:`PyObject_SetArenaAllocator` - * :c:func:`Py_SetProgramName` - * :c:func:`Py_SetPythonHome` - * :c:func:`PySys_ResetWarnOptions` - * the configuration functions covered in :ref:`init-config` - -* Informative functions: - - * :c:func:`Py_IsInitialized` - * :c:func:`PyMem_GetAllocator` - * :c:func:`PyObject_GetArenaAllocator` - * :c:func:`Py_GetBuildInfo` - * :c:func:`Py_GetCompiler` - * :c:func:`Py_GetCopyright` - * :c:func:`Py_GetPlatform` - * :c:func:`Py_GetVersion` - * :c:func:`Py_IsInitialized` - -* Utilities: - - * :c:func:`Py_DecodeLocale` - * the status reporting and utility functions covered in :ref:`init-config` - -* Memory allocators: - - * :c:func:`PyMem_RawMalloc` - * :c:func:`PyMem_RawRealloc` - * :c:func:`PyMem_RawCalloc` - * :c:func:`PyMem_RawFree` - -* Synchronization: - - * :c:func:`PyMutex_Lock` - * :c:func:`PyMutex_Unlock` - -.. note:: - - Despite their apparent similarity to some of the functions listed above, - the following functions **should not be called** before the interpreter has - been initialized: :c:func:`Py_EncodeLocale`, :c:func:`PyEval_InitThreads`, and - :c:func:`Py_RunMain`. - - -.. _global-conf-vars: - -Global configuration variables -============================== - -Python has variables for the global configuration to control different features -and options. By default, these flags are controlled by :ref:`command line -options `. - -When a flag is set by an option, the value of the flag is the number of times -that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag` -to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. - -.. c:var:: int Py_BytesWarningFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.bytes_warning` should be used instead, see :ref:`Python - Initialization Configuration `. - - Issue a warning when comparing :class:`bytes` or :class:`bytearray` with - :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater - or equal to ``2``. - - Set by the :option:`-b` option. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_DebugFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.parser_debug` should be used instead, see :ref:`Python - Initialization Configuration `. - - Turn on parser debugging output (for expert only, depending on compilation - options). - - Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_DontWriteBytecodeFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.write_bytecode` should be used instead, see :ref:`Python - Initialization Configuration `. - - If set to non-zero, Python won't try to write ``.pyc`` files on the - import of source modules. - - Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` - environment variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_FrozenFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.pathconfig_warnings` should be used instead, see - :ref:`Python Initialization Configuration `. - - Private flag used by ``_freeze_module`` and ``frozenmain`` programs. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_HashRandomizationFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.hash_seed` and :c:member:`PyConfig.use_hash_seed` should - be used instead, see :ref:`Python Initialization Configuration - `. - - Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to - a non-empty string. - - If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment - variable to initialize the secret hash seed. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_IgnoreEnvironmentFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.use_environment` should be used instead, see - :ref:`Python Initialization Configuration `. - - Ignore all :envvar:`!PYTHON*` environment variables, e.g. - :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. - - Set by the :option:`-E` and :option:`-I` options. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_InspectFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.inspect` should be used instead, see - :ref:`Python Initialization Configuration `. - - When a script is passed as first argument or the :option:`-c` option is used, - enter interactive mode after executing the script or the command, even when - :data:`sys.stdin` does not appear to be a terminal. - - Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_InteractiveFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.interactive` should be used instead, see - :ref:`Python Initialization Configuration `. - - Set by the :option:`-i` option. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_IsolatedFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.isolated` should be used instead, see - :ref:`Python Initialization Configuration `. - - Run Python in isolated mode. In isolated mode :data:`sys.path` contains - neither the script's directory nor the user's site-packages directory. - - Set by the :option:`-I` option. - - .. versionadded:: 3.4 - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_LegacyWindowsFSEncodingFlag - - This API is kept for backward compatibility: setting - :c:member:`PyPreConfig.legacy_windows_fs_encoding` should be used instead, see - :ref:`Python Initialization Configuration `. - - If the flag is non-zero, use the ``mbcs`` encoding with ``replace`` error - handler, instead of the UTF-8 encoding with ``surrogatepass`` error handler, - for the :term:`filesystem encoding and error handler`. - - Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment - variable is set to a non-empty string. - - See :pep:`529` for more details. - - .. availability:: Windows. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_LegacyWindowsStdioFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.legacy_windows_stdio` should be used instead, see - :ref:`Python Initialization Configuration `. - - If the flag is non-zero, use :class:`io.FileIO` instead of - :class:`!io._WindowsConsoleIO` for :mod:`sys` standard streams. - - Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment - variable is set to a non-empty string. - - See :pep:`528` for more details. - - .. availability:: Windows. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_NoSiteFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.site_import` should be used instead, see - :ref:`Python Initialization Configuration `. - - Disable the import of the module :mod:`site` and the site-dependent - manipulations of :data:`sys.path` that it entails. Also disable these - manipulations if :mod:`site` is explicitly imported later (call - :func:`site.main` if you want them to be triggered). - - Set by the :option:`-S` option. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_NoUserSiteDirectory - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.user_site_directory` should be used instead, see - :ref:`Python Initialization Configuration `. - - Don't add the :data:`user site-packages directory ` to - :data:`sys.path`. - - Set by the :option:`-s` and :option:`-I` options, and the - :envvar:`PYTHONNOUSERSITE` environment variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_OptimizeFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.optimization_level` should be used instead, see - :ref:`Python Initialization Configuration `. - - Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_QuietFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.quiet` should be used instead, see :ref:`Python - Initialization Configuration `. - - Don't display the copyright and version messages even in interactive mode. - - Set by the :option:`-q` option. - - .. versionadded:: 3.2 - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_UnbufferedStdioFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.buffered_stdio` should be used instead, see :ref:`Python - Initialization Configuration `. - - Force the stdout and stderr streams to be unbuffered. - - Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` - environment variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_VerboseFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.verbose` should be used instead, see :ref:`Python - Initialization Configuration `. - - Print a message each time a module is initialized, showing the place - (filename or built-in module) from which it is loaded. If greater or equal - to ``2``, print a message for each file that is checked for when - searching for a module. Also provides information on module cleanup at exit. - - Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - - -Initializing and finalizing the interpreter -=========================================== - - -.. c:function:: void Py_Initialize() - - .. index:: - single: PyEval_InitThreads() - single: modules (in module sys) - single: path (in module sys) - pair: module; builtins - pair: module; __main__ - pair: module; sys - triple: module; search; path - single: Py_FinalizeEx (C function) - - Initialize the Python interpreter. In an application embedding Python, - this should be called before using any other Python/C API functions; see - :ref:`Before Python Initialization ` for the few exceptions. - - This initializes the table of loaded modules (``sys.modules``), and creates - the fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. - It also initializes the module search path (``sys.path``). It does not set - ``sys.argv``; use the :ref:`Python Initialization Configuration ` - API for that. This is a no-op when called for a second time (without calling - :c:func:`Py_FinalizeEx` first). There is no return value; it is a fatal - error if the initialization fails. - - Use :c:func:`Py_InitializeFromConfig` to customize the - :ref:`Python Initialization Configuration `. - - .. note:: - On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, - which will also affect non-Python uses of the console using the C Runtime. - - -.. c:function:: void Py_InitializeEx(int initsigs) - - This function works like :c:func:`Py_Initialize` if *initsigs* is ``1``. If - *initsigs* is ``0``, it skips initialization registration of signal handlers, - which may be useful when CPython is embedded as part of a larger application. - - Use :c:func:`Py_InitializeFromConfig` to customize the - :ref:`Python Initialization Configuration `. - - -.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) - - Initialize Python from *config* configuration, as described in - :ref:`init-from-config`. - - See the :ref:`init-config` section for details on pre-initializing the - interpreter, populating the runtime configuration structure, and querying - the returned status structure. - - -.. c:function:: int Py_IsInitialized() - - Return true (nonzero) when the Python interpreter has been initialized, false - (zero) if not. After :c:func:`Py_FinalizeEx` is called, this returns false until - :c:func:`Py_Initialize` is called again. - - -.. c:function:: int Py_IsFinalizing() - - Return true (non-zero) if the main Python interpreter is - :term:`shutting down `. Return false (zero) otherwise. - - .. versionadded:: 3.13 - - -.. c:function:: int Py_FinalizeEx() - - Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of - Python/C API functions, and destroy all sub-interpreters (see - :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since - the last call to :c:func:`Py_Initialize`. This is a no-op when called for a second - time (without calling :c:func:`Py_Initialize` again first). - - Since this is the reverse of :c:func:`Py_Initialize`, it should be called - in the same thread with the same interpreter active. That means - the main thread and the main interpreter. - This should never be called while :c:func:`Py_RunMain` is running. - - Normally the return value is ``0``. - If there were errors during finalization (flushing buffered data), - ``-1`` is returned. - - Note that Python will do a best effort at freeing all memory allocated by the Python - interpreter. Therefore, any C-Extension should make sure to correctly clean up all - of the preveiously allocated PyObjects before using them in subsequent calls to - :c:func:`Py_Initialize`. Otherwise it could introduce vulnerabilities and incorrect - behavior. - - This function is provided for a number of reasons. An embedding application - might want to restart Python without having to restart the application itself. - An application that has loaded the Python interpreter from a dynamically - loadable library (or DLL) might want to free all memory allocated by Python - before unloading the DLL. During a hunt for memory leaks in an application a - developer might want to free all memory allocated by Python before exiting from - the application. - - **Bugs and caveats:** The destruction of modules and objects in modules is done - in random order; this may cause destructors (:meth:`~object.__del__` methods) to fail - when they depend on other objects (even functions) or modules. Dynamically - loaded extension modules loaded by Python are not unloaded. Small amounts of - memory allocated by the Python interpreter may not be freed (if you find a leak, - please report it). Memory tied up in circular references between objects is not - freed. Interned strings will all be deallocated regardless of their reference count. - Some memory allocated by extension modules may not be freed. Some extensions may not - work properly if their initialization routine is called more than once; this can - happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` - more than once. :c:func:`Py_FinalizeEx` must not be called recursively from - within itself. Therefore, it must not be called by any code that may be run - as part of the interpreter shutdown process, such as :py:mod:`atexit` - handlers, object finalizers, or any code that may be run while flushing the - stdout and stderr files. - - .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx - - .. versionadded:: 3.6 - - -.. c:function:: void Py_Finalize() - - This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that - disregards the return value. - - -.. c:function:: int Py_BytesMain(int argc, char **argv) - - Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings, - allowing the calling application to delegate the text decoding step to - the CPython runtime. - - .. versionadded:: 3.8 - - -.. c:function:: int Py_Main(int argc, wchar_t **argv) - - The main program for the standard interpreter, encapsulating a full - initialization/finalization cycle, as well as additional - behaviour to implement reading configurations settings from the environment - and command line, and then executing ``__main__`` in accordance with - :ref:`using-on-cmdline`. - - This is made available for programs which wish to support the full CPython - command line interface, rather than just embedding a Python runtime in a - larger application. - - The *argc* and *argv* parameters are similar to those which are passed to a - C program's :c:func:`main` function, except that the *argv* entries are first - converted to ``wchar_t`` using :c:func:`Py_DecodeLocale`. It is also - important to note that the argument list entries may be modified to point to - strings other than those passed in (however, the contents of the strings - pointed to by the argument list are not modified). - - The return value is ``2`` if the argument list does not represent a valid - Python command line, and otherwise the same as :c:func:`Py_RunMain`. - - In terms of the CPython runtime configuration APIs documented in the - :ref:`runtime configuration ` section (and without accounting - for error handling), ``Py_Main`` is approximately equivalent to:: - - PyConfig config; - PyConfig_InitPythonConfig(&config); - PyConfig_SetArgv(&config, argc, argv); - Py_InitializeFromConfig(&config); - PyConfig_Clear(&config); - - Py_RunMain(); - - In normal usage, an embedding application will call this function - *instead* of calling :c:func:`Py_Initialize`, :c:func:`Py_InitializeEx` or - :c:func:`Py_InitializeFromConfig` directly, and all settings will be applied - as described elsewhere in this documentation. If this function is instead - called *after* a preceding runtime initialization API call, then exactly - which environmental and command line configuration settings will be updated - is version dependent (as it depends on which settings correctly support - being modified after they have already been set once when the runtime was - first initialized). - - -.. c:function:: int Py_RunMain(void) - - Executes the main module in a fully configured CPython runtime. - - Executes the command (:c:member:`PyConfig.run_command`), the script - (:c:member:`PyConfig.run_filename`) or the module - (:c:member:`PyConfig.run_module`) specified on the command line or in the - configuration. If none of these values are set, runs the interactive Python - prompt (REPL) using the ``__main__`` module's global namespace. - - If :c:member:`PyConfig.inspect` is not set (the default), the return value - will be ``0`` if the interpreter exits normally (that is, without raising - an exception), the exit status of an unhandled :exc:`SystemExit`, or ``1`` - for any other unhandled exception. - - If :c:member:`PyConfig.inspect` is set (such as when the :option:`-i` option - is used), rather than returning when the interpreter exits, execution will - instead resume in an interactive Python prompt (REPL) using the ``__main__`` - module's global namespace. If the interpreter exited with an exception, it - is immediately raised in the REPL session. The function return value is - then determined by the way the *REPL session* terminates: ``0``, ``1``, or - the status of a :exc:`SystemExit`, as specified above. - - This function always finalizes the Python interpreter before it returns. - - See :ref:`Python Configuration ` for an example of a - customized Python that always runs in isolated mode using - :c:func:`Py_RunMain`. - -.. c:function:: int PyUnstable_AtExit(PyInterpreterState *interp, void (*func)(void *), void *data) - - Register an :mod:`atexit` callback for the target interpreter *interp*. - This is similar to :c:func:`Py_AtExit`, but takes an explicit interpreter and - data pointer for the callback. - - There must be an :term:`attached thread state` for *interp*. - - .. versionadded:: 3.13 - -Process-wide parameters -======================= - - -.. c:function:: void Py_SetProgramName(const wchar_t *name) - - .. index:: - single: Py_Initialize() - single: main() - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.program_name` should be used instead, see :ref:`Python - Initialization Configuration `. - - This function should be called before :c:func:`Py_Initialize` is called for - the first time, if it is called at all. It tells the interpreter the value - of the ``argv[0]`` argument to the :c:func:`main` function of the program - (converted to wide characters). - This is used by some other functions below to find - the Python run-time libraries relative to the interpreter executable. The - default value is ``'python'``. The argument should point to a - zero-terminated wide character string in static storage whose contents will not - change for the duration of the program's execution. No code in the Python - interpreter will change the contents of this storage. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - .. deprecated-removed:: 3.11 3.15 - - -.. c:function:: const char* Py_GetVersion() - - Return the version of this Python interpreter. This is a string that looks - something like :: - - "3.0a5+ (py3k:63103M, May 12 2008, 00:53:55) \n[GCC 4.2.3]" - - .. index:: single: version (in module sys) - - The first word (up to the first space character) is the current Python version; - the first characters are the major and minor version separated by a - period. The returned string points into static storage; the caller should not - modify its value. The value is available to Python code as :data:`sys.version`. - - See also the :c:var:`Py_Version` constant. - - -.. c:function:: const char* Py_GetPlatform() - - .. index:: single: platform (in module sys) - - Return the platform identifier for the current platform. On Unix, this is - formed from the "official" name of the operating system, converted to lower - case, followed by the major revision number; e.g., for Solaris 2.x, which is - also known as SunOS 5.x, the value is ``'sunos5'``. On macOS, it is - ``'darwin'``. On Windows, it is ``'win'``. The returned string points into - static storage; the caller should not modify its value. The value is available - to Python code as ``sys.platform``. - - -.. c:function:: const char* Py_GetCopyright() - - Return the official copyright string for the current Python version, for example - - ``'Copyright 1991-1995 Stichting Mathematisch Centrum, Amsterdam'`` - - .. index:: single: copyright (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as ``sys.copyright``. - - -.. c:function:: const char* Py_GetCompiler() - - Return an indication of the compiler used to build the current Python version, - in square brackets, for example:: - - "[GCC 2.7.2.2]" - - .. index:: single: version (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as part of the variable - ``sys.version``. - - -.. c:function:: const char* Py_GetBuildInfo() - - Return information about the sequence number and build date and time of the - current Python interpreter instance, for example :: - - "#67, Aug 1 1997, 22:34:28" - - .. index:: single: version (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as part of the variable - ``sys.version``. - - -.. c:function:: void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) - - .. index:: - single: main() - single: Py_FatalError() - single: argv (in module sys) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.argv`, :c:member:`PyConfig.parse_argv` and - :c:member:`PyConfig.safe_path` should be used instead, see :ref:`Python - Initialization Configuration `. - - Set :data:`sys.argv` based on *argc* and *argv*. These parameters are - similar to those passed to the program's :c:func:`main` function with the - difference that the first entry should refer to the script file to be - executed rather than the executable hosting the Python interpreter. If there - isn't a script that will be run, the first entry in *argv* can be an empty - string. If this function fails to initialize :data:`sys.argv`, a fatal - condition is signalled using :c:func:`Py_FatalError`. - - If *updatepath* is zero, this is all the function does. If *updatepath* - is non-zero, the function also modifies :data:`sys.path` according to the - following algorithm: - - - If the name of an existing script is passed in ``argv[0]``, the absolute - path of the directory where the script is located is prepended to - :data:`sys.path`. - - Otherwise (that is, if *argc* is ``0`` or ``argv[0]`` doesn't point - to an existing file name), an empty string is prepended to - :data:`sys.path`, which is the same as prepending the current working - directory (``"."``). - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` - members of the :ref:`Python Initialization Configuration `. - - .. note:: - It is recommended that applications embedding the Python interpreter - for purposes other than executing a single script pass ``0`` as *updatepath*, - and update :data:`sys.path` themselves if desired. - See :cve:`2008-5983`. - - On versions before 3.1.3, you can achieve the same effect by manually - popping the first :data:`sys.path` element after having called - :c:func:`PySys_SetArgv`, for example using:: - - PyRun_SimpleString("import sys; sys.path.pop(0)\n"); - - .. versionadded:: 3.1.3 - - .. XXX impl. doesn't seem consistent in allowing ``0``/``NULL`` for the params; - check w/ Guido. - - .. deprecated-removed:: 3.11 3.15 - - -.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.argv` and :c:member:`PyConfig.parse_argv` should be used - instead, see :ref:`Python Initialization Configuration `. - - This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set - to ``1`` unless the :program:`python` interpreter was started with the - :option:`-I`. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` - members of the :ref:`Python Initialization Configuration `. - - .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. - - .. deprecated-removed:: 3.11 3.15 - - -.. c:function:: void Py_SetPythonHome(const wchar_t *home) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.home` should be used instead, see :ref:`Python - Initialization Configuration `. - - Set the default "home" directory, that is, the location of the standard - Python libraries. See :envvar:`PYTHONHOME` for the meaning of the - argument string. - - The argument should point to a zero-terminated character string in static - storage whose contents will not change for the duration of the program's - execution. No code in the Python interpreter will change the contents of - this storage. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - .. deprecated-removed:: 3.11 3.15 - - -.. _threads: - -Thread State and the Global Interpreter Lock -============================================ - -.. index:: - single: global interpreter lock - single: interpreter lock - single: lock, interpreter - -Unless on a :term:`free-threaded ` build of :term:`CPython`, -the Python interpreter is not fully thread-safe. In order to support -multi-threaded Python programs, there's a global lock, called the :term:`global -interpreter lock` or :term:`GIL`, that must be held by the current thread before -it can safely access Python objects. Without the lock, even the simplest -operations could cause problems in a multi-threaded program: for example, when -two threads simultaneously increment the reference count of the same object, the -reference count could end up being incremented only once instead of twice. - -.. index:: single: setswitchinterval (in module sys) - -Therefore, the rule exists that only the thread that has acquired the -:term:`GIL` may operate on Python objects or call Python/C API functions. -In order to emulate concurrency of execution, the interpreter regularly -tries to switch threads (see :func:`sys.setswitchinterval`). The lock is also -released around potentially blocking I/O operations like reading or writing -a file, so that other Python threads can run in the meantime. - -.. index:: - single: PyThreadState (C type) - -The Python interpreter keeps some thread-specific bookkeeping information -inside a data structure called :c:type:`PyThreadState`, known as a :term:`thread state`. -Each OS thread has a thread-local pointer to a :c:type:`PyThreadState`; a thread state -referenced by this pointer is considered to be :term:`attached `. - -A thread can only have one :term:`attached thread state` at a time. An attached -thread state is typically analogous with holding the :term:`GIL`, except on -:term:`free-threaded ` builds. On builds with the :term:`GIL` enabled, -:term:`attaching ` a thread state will block until the :term:`GIL` -can be acquired. However, even on builds with the :term:`GIL` disabled, it is still required -to have an attached thread state to call most of the C API. - -In general, there will always be an :term:`attached thread state` when using Python's C API. -Only in some specific cases (such as in a :c:macro:`Py_BEGIN_ALLOW_THREADS` block) will the -thread not have an attached thread state. If uncertain, check if :c:func:`PyThreadState_GetUnchecked` returns -``NULL``. - -Detaching the thread state from extension code ----------------------------------------------- - -Most extension code manipulating the :term:`thread state` has the following simple -structure:: - - Save the thread state in a local variable. - ... Do some blocking I/O operation ... - Restore the thread state from the local variable. - -This is so common that a pair of macros exists to simplify it:: - - Py_BEGIN_ALLOW_THREADS - ... Do some blocking I/O operation ... - Py_END_ALLOW_THREADS - -.. index:: - single: Py_BEGIN_ALLOW_THREADS (C macro) - single: Py_END_ALLOW_THREADS (C macro) - -The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a -hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the -block. - -The block above expands to the following code:: - - PyThreadState *_save; - - _save = PyEval_SaveThread(); - ... Do some blocking I/O operation ... - PyEval_RestoreThread(_save); - -.. index:: - single: PyEval_RestoreThread (C function) - single: PyEval_SaveThread (C function) - -Here is how these functions work: - -The :term:`attached thread state` holds the :term:`GIL` for the entire interpreter. When detaching -the :term:`attached thread state`, the :term:`GIL` is released, allowing other threads to attach -a thread state to their own thread, thus getting the :term:`GIL` and can start executing. -The pointer to the prior :term:`attached thread state` is stored as a local variable. -Upon reaching :c:macro:`Py_END_ALLOW_THREADS`, the thread state that was -previously :term:`attached ` is passed to :c:func:`PyEval_RestoreThread`. -This function will block until another releases its :term:`thread state `, -thus allowing the old :term:`thread state ` to get re-attached and the -C API can be called again. - -For :term:`free-threaded ` builds, the :term:`GIL` is normally -out of the question, but detaching the :term:`thread state ` is still required -for blocking I/O and long operations. The difference is that threads don't have to wait for the :term:`GIL` -to be released to attach their thread state, allowing true multi-core parallelism. - -.. note:: - Calling system I/O functions is the most common use case for detaching - the :term:`thread state `, but it can also be useful before calling - long-running computations which don't need access to Python objects, such - as compression or cryptographic functions operating over memory buffers. - For example, the standard :mod:`zlib` and :mod:`hashlib` modules detach the - :term:`thread state ` when compressing or hashing data. - - -.. _gilstate: - -Non-Python created threads --------------------------- - -When threads are created using the dedicated Python APIs (such as the -:mod:`threading` module), a thread state is automatically associated to them -and the code showed above is therefore correct. However, when threads are -created from C (for example by a third-party library with its own thread -management), they don't hold the :term:`GIL`, because they don't have an -:term:`attached thread state`. - -If you need to call Python code from these threads (often this will be part -of a callback API provided by the aforementioned third-party library), -you must first register these threads with the interpreter by -creating an :term:`attached thread state` before you can start using the Python/C -API. When you are done, you should detach the :term:`thread state `, and -finally free it. - -The :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` functions do -all of the above automatically. The typical idiom for calling into Python -from a C thread is:: - - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); - - /* Perform Python actions here. */ - result = CallSomeFunction(); - /* evaluate result or handle exception */ - - /* Release the thread. No Python API allowed beyond this point. */ - PyGILState_Release(gstate); - -Note that the ``PyGILState_*`` functions assume there is only one global -interpreter (created automatically by :c:func:`Py_Initialize`). Python -supports the creation of additional interpreters (using -:c:func:`Py_NewInterpreter`), but mixing multiple interpreters and the -``PyGILState_*`` API is unsupported. This is because :c:func:`PyGILState_Ensure` -and similar functions default to :term:`attaching ` a -:term:`thread state` for the main interpreter, meaning that the thread can't safely -interact with the calling subinterpreter. - -Supporting subinterpreters in non-Python threads ------------------------------------------------- - -If you would like to support subinterpreters with non-Python created threads, you -must use the ``PyThreadState_*`` API instead of the traditional ``PyGILState_*`` -API. - -In particular, you must store the interpreter state from the calling -function and pass it to :c:func:`PyThreadState_New`, which will ensure that -the :term:`thread state` is targeting the correct interpreter:: - - /* The return value of PyInterpreterState_Get() from the - function that created this thread. */ - PyInterpreterState *interp = ThreadData->interp; - PyThreadState *tstate = PyThreadState_New(interp); - PyThreadState_Swap(tstate); - - /* GIL of the subinterpreter is now held. - Perform Python actions here. */ - result = CallSomeFunction(); - /* evaluate result or handle exception */ - - /* Destroy the thread state. No Python API allowed beyond this point. */ - PyThreadState_Clear(tstate); - PyThreadState_DeleteCurrent(); - -.. _fork-and-threads: - -Cautions about fork() ---------------------- - -Another important thing to note about threads is their behaviour in the face -of the C :c:func:`fork` call. On most systems with :c:func:`fork`, after a -process forks only the thread that issued the fork will exist. This has a -concrete impact both on how locks must be handled and on all stored state -in CPython's runtime. - -The fact that only the "current" thread remains -means any locks held by other threads will never be released. Python solves -this for :func:`os.fork` by acquiring the locks it uses internally before -the fork, and releasing them afterwards. In addition, it resets any -:ref:`lock-objects` in the child. When extending or embedding Python, there -is no way to inform Python of additional (non-Python) locks that need to be -acquired before or reset after a fork. OS facilities such as -:c:func:`!pthread_atfork` would need to be used to accomplish the same thing. -Additionally, when extending or embedding Python, calling :c:func:`fork` -directly rather than through :func:`os.fork` (and returning to or calling -into Python) may result in a deadlock by one of Python's internal locks -being held by a thread that is defunct after the fork. -:c:func:`PyOS_AfterFork_Child` tries to reset the necessary locks, but is not -always able to. - -The fact that all other threads go away also means that CPython's -runtime state there must be cleaned up properly, which :func:`os.fork` -does. This means finalizing all other :c:type:`PyThreadState` objects -belonging to the current interpreter and all other -:c:type:`PyInterpreterState` objects. Due to this and the special -nature of the :ref:`"main" interpreter `, -:c:func:`fork` should only be called in that interpreter's "main" -thread, where the CPython global runtime was originally initialized. -The only exception is if :c:func:`exec` will be called immediately -after. - -.. _cautions-regarding-runtime-finalization: - -Cautions regarding runtime finalization ---------------------------------------- - -In the late stage of :term:`interpreter shutdown`, after attempting to wait for -non-daemon threads to exit (though this can be interrupted by -:class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime -is marked as *finalizing*: :c:func:`Py_IsFinalizing` and -:func:`sys.is_finalizing` return true. At this point, only the *finalization -thread* that initiated finalization (typically the main thread) is allowed to -acquire the :term:`GIL`. - -If any thread, other than the finalization thread, attempts to attach a :term:`thread state` -during finalization, either explicitly or -implicitly, the thread enters **a permanently blocked state** -where it remains until the program exits. In most cases this is harmless, but this can result -in deadlock if a later stage of finalization attempts to acquire a lock owned by the -blocked thread, or otherwise waits on the blocked thread. - -Gross? Yes. This prevents random crashes and/or unexpectedly skipped C++ -finalizations further up the call stack when such threads were forcibly exited -here in CPython 3.13 and earlier. The CPython runtime :term:`thread state` C APIs -have never had any error reporting or handling expectations at :term:`thread state` -attachment time that would've allowed for graceful exit from this situation. Changing that -would require new stable C APIs and rewriting the majority of C code in the -CPython ecosystem to use those with error handling. - - -High-level API --------------- - -These are the most commonly used types and functions when writing C extension -code, or when embedding the Python interpreter: - -.. c:type:: PyInterpreterState - - This data structure represents the state shared by a number of cooperating - threads. Threads belonging to the same interpreter share their module - administration and a few other internal items. There are no public members in - this structure. - - Threads belonging to different interpreters initially share nothing, except - process state like available memory, open file descriptors and such. The global - interpreter lock is also shared by all threads, regardless of to which - interpreter they belong. - - .. versionchanged:: 3.12 - - :pep:`684` introduced the possibility - of a :ref:`per-interpreter GIL `. - See :c:func:`Py_NewInterpreterFromConfig`. - - -.. c:type:: PyThreadState - - This data structure represents the state of a single thread. The only public - data member is: - - .. c:member:: PyInterpreterState *interp - - This thread's interpreter state. - - -.. c:function:: void PyEval_InitThreads() - - .. index:: - single: PyEval_AcquireThread() - single: PyEval_ReleaseThread() - single: PyEval_SaveThread() - single: PyEval_RestoreThread() - - Deprecated function which does nothing. - - In Python 3.6 and older, this function created the GIL if it didn't exist. - - .. versionchanged:: 3.9 - The function now does nothing. - - .. versionchanged:: 3.7 - This function is now called by :c:func:`Py_Initialize()`, so you don't - have to call it yourself anymore. - - .. versionchanged:: 3.2 - This function cannot be called before :c:func:`Py_Initialize()` anymore. - - .. deprecated:: 3.9 - - .. index:: pair: module; _thread - - -.. c:function:: PyThreadState* PyEval_SaveThread() - - Detach the :term:`attached thread state` and return it. - The thread will have no :term:`thread state` upon returning. - - -.. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) - - Set the :term:`attached thread state` to *tstate*. - The passed :term:`thread state` **should not** be :term:`attached `, - otherwise deadlock ensues. *tstate* will be attached upon returning. - - .. note:: - Calling this function from a thread when the runtime is finalizing will - hang the thread until the program exits, even if the thread was not - created by Python. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - -.. c:function:: PyThreadState* PyThreadState_Get() - - Return the :term:`attached thread state`. If the thread has no attached - thread state, (such as when inside of :c:macro:`Py_BEGIN_ALLOW_THREADS` - block), then this issues a fatal error (so that the caller needn't check - for ``NULL``). - - See also :c:func:`PyThreadState_GetUnchecked`. - -.. c:function:: PyThreadState* PyThreadState_GetUnchecked() - - Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a - fatal error if it is NULL. The caller is responsible to check if the result - is NULL. - - .. versionadded:: 3.13 - In Python 3.5 to 3.12, the function was private and known as - ``_PyThreadState_UncheckedGet()``. - - -.. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) - - Set the :term:`attached thread state` to *tstate*, and return the - :term:`thread state` that was attached prior to calling. - - This function is safe to call without an :term:`attached thread state`; it - will simply return ``NULL`` indicating that there was no prior thread state. - - .. seealso: - :c:func:`PyEval_ReleaseThread` - - .. note:: - Similar to :c:func:`PyGILState_Ensure`, this function will hang the - thread if the runtime is finalizing. - - -The following functions use thread-local storage, and are not compatible -with sub-interpreters: - -.. c:function:: PyGILState_STATE PyGILState_Ensure() - - Ensure that the current thread is ready to call the Python C API regardless - of the current state of Python, or of the :term:`attached thread state`. This may - be called as many times as desired by a thread as long as each call is - matched with a call to :c:func:`PyGILState_Release`. In general, other - thread-related APIs may be used between :c:func:`PyGILState_Ensure` and - :c:func:`PyGILState_Release` calls as long as the thread state is restored to - its previous state before the Release(). For example, normal usage of the - :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is - acceptable. - - The return value is an opaque "handle" to the :term:`attached thread state` when - :c:func:`PyGILState_Ensure` was called, and must be passed to - :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even - though recursive calls are allowed, these handles *cannot* be shared - each - unique call to :c:func:`PyGILState_Ensure` must save the handle for its call - to :c:func:`PyGILState_Release`. - - When the function returns, there will be an :term:`attached thread state` - and the thread will be able to call arbitrary Python code. Failure is a fatal error. - - .. warning:: - Calling this function when the runtime is finalizing is unsafe. Doing - so will either hang the thread until the program ends, or fully crash - the interpreter in rare cases. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - -.. c:function:: void PyGILState_Release(PyGILState_STATE) - - Release any resources previously acquired. After this call, Python's state will - be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call - (but generally this state will be unknown to the caller, hence the use of the - GILState API). - - Every call to :c:func:`PyGILState_Ensure` must be matched by a call to - :c:func:`PyGILState_Release` on the same thread. - -.. c:function:: PyThreadState* PyGILState_GetThisThreadState() - - Get the :term:`attached thread state` for this thread. May return ``NULL`` if no - GILState API has been used on the current thread. Note that the main thread - always has such a thread-state, even if no auto-thread-state call has been - made on the main thread. This is mainly a helper/diagnostic function. - - .. note:: - This function does not account for :term:`thread states ` created - by something other than :c:func:`PyGILState_Ensure` (such as :c:func:`PyThreadState_New`). - Prefer :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` - for most cases. - - .. seealso: :c:func:`PyThreadState_Get`` - -.. c:function:: int PyGILState_Check() - - Return ``1`` if the current thread is holding the :term:`GIL` and ``0`` otherwise. - This function can be called from any thread at any time. - Only if it has had its :term:`thread state ` initialized - via :c:func:`PyGILState_Ensure` will it return ``1``. - This is mainly a helper/diagnostic function. It can be useful - for example in callback contexts or memory allocation functions when - knowing that the :term:`GIL` is locked can allow the caller to perform sensitive - actions or otherwise behave differently. - - .. note:: - If the current Python process has ever created a subinterpreter, this - function will *always* return ``1``. Prefer :c:func:`PyThreadState_GetUnchecked` - for most cases. - - .. versionadded:: 3.4 - - -The following macros are normally used without a trailing semicolon; look for -example usage in the Python source distribution. - - -.. c:macro:: Py_BEGIN_ALLOW_THREADS - - This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. - Note that it contains an opening brace; it must be matched with a following - :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this - macro. - - -.. c:macro:: Py_END_ALLOW_THREADS - - This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains - a closing brace; it must be matched with an earlier - :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of - this macro. - - -.. c:macro:: Py_BLOCK_THREADS - - This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to - :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. - - -.. c:macro:: Py_UNBLOCK_THREADS - - This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to - :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable - declaration. - - -Low-level API -------------- - -All of the following functions must be called after :c:func:`Py_Initialize`. - -.. versionchanged:: 3.7 - :c:func:`Py_Initialize()` now initializes the :term:`GIL` - and sets an :term:`attached thread state`. - - -.. c:function:: PyInterpreterState* PyInterpreterState_New() - - Create a new interpreter state object. An :term:`attached thread state` is not needed, - but may optionally exist if it is necessary to serialize calls to this - function. - - .. audit-event:: cpython.PyInterpreterState_New "" c.PyInterpreterState_New - - -.. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp) - - Reset all information in an interpreter state object. There must be - an :term:`attached thread state` for the interpreter. - - .. audit-event:: cpython.PyInterpreterState_Clear "" c.PyInterpreterState_Clear - - -.. c:function:: void PyInterpreterState_Delete(PyInterpreterState *interp) - - Destroy an interpreter state object. There **should not** be an - :term:`attached thread state` for the target interpreter. The interpreter - state must have been reset with a previous call to :c:func:`PyInterpreterState_Clear`. - - -.. c:function:: PyThreadState* PyThreadState_New(PyInterpreterState *interp) - - Create a new thread state object belonging to the given interpreter object. - An :term:`attached thread state` is not needed. - -.. c:function:: void PyThreadState_Clear(PyThreadState *tstate) - - Reset all information in a :term:`thread state` object. *tstate* - must be :term:`attached ` - - .. versionchanged:: 3.9 - This function now calls the :c:member:`PyThreadState.on_delete` callback. - Previously, that happened in :c:func:`PyThreadState_Delete`. - - .. versionchanged:: 3.13 - The :c:member:`PyThreadState.on_delete` callback was removed. - - -.. c:function:: void PyThreadState_Delete(PyThreadState *tstate) - - Destroy a :term:`thread state` object. *tstate* should not - be :term:`attached ` to any thread. - *tstate* must have been reset with a previous call to - :c:func:`PyThreadState_Clear`. - - -.. c:function:: void PyThreadState_DeleteCurrent(void) - - Detach the :term:`attached thread state` (which must have been reset - with a previous call to :c:func:`PyThreadState_Clear`) and then destroy it. - - No :term:`thread state` will be :term:`attached ` upon - returning. - -.. c:function:: PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) - - Get the current frame of the Python thread state *tstate*. - - Return a :term:`strong reference`. Return ``NULL`` if no frame is currently - executing. - - See also :c:func:`PyEval_GetFrame`. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: uint64_t PyThreadState_GetID(PyThreadState *tstate) - - Get the unique :term:`thread state` identifier of the Python thread state *tstate*. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: PyInterpreterState* PyThreadState_GetInterpreter(PyThreadState *tstate) - - Get the interpreter of the Python thread state *tstate*. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: void PyThreadState_EnterTracing(PyThreadState *tstate) - - Suspend tracing and profiling in the Python thread state *tstate*. - - Resume them using the :c:func:`PyThreadState_LeaveTracing` function. - - .. versionadded:: 3.11 - - -.. c:function:: void PyThreadState_LeaveTracing(PyThreadState *tstate) - - Resume tracing and profiling in the Python thread state *tstate* suspended - by the :c:func:`PyThreadState_EnterTracing` function. - - See also :c:func:`PyEval_SetTrace` and :c:func:`PyEval_SetProfile` - functions. - - .. versionadded:: 3.11 - - -.. c:function:: PyInterpreterState* PyInterpreterState_Get(void) - - Get the current interpreter. - - Issue a fatal error if there no :term:`attached thread state`. - It cannot return NULL. - - .. versionadded:: 3.9 - - -.. c:function:: int64_t PyInterpreterState_GetID(PyInterpreterState *interp) - - Return the interpreter's unique ID. If there was any error in doing - so then ``-1`` is returned and an error is set. - - The caller must have an :term:`attached thread state`. - - .. versionadded:: 3.7 - - -.. c:function:: PyObject* PyInterpreterState_GetDict(PyInterpreterState *interp) - - Return a dictionary in which interpreter-specific data may be stored. - If this function returns ``NULL`` then no exception has been raised and - the caller should assume no interpreter-specific dict is available. - - This is not a replacement for :c:func:`PyModule_GetState()`, which - extensions should use to store interpreter-specific state information. - - .. versionadded:: 3.8 - - -.. c:type:: PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) - - Type of a frame evaluation function. - - The *throwflag* parameter is used by the ``throw()`` method of generators: - if non-zero, handle the current exception. - - .. versionchanged:: 3.9 - The function now takes a *tstate* parameter. - - .. versionchanged:: 3.11 - The *frame* parameter changed from ``PyFrameObject*`` to ``_PyInterpreterFrame*``. - -.. c:function:: _PyFrameEvalFunction _PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) - - Get the frame evaluation function. - - See the :pep:`523` "Adding a frame evaluation API to CPython". - - .. versionadded:: 3.9 - -.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) - - Set the frame evaluation function. - - See the :pep:`523` "Adding a frame evaluation API to CPython". - - .. versionadded:: 3.9 - - -.. c:function:: PyObject* PyThreadState_GetDict() - - Return a dictionary in which extensions can store thread-specific state - information. Each extension should use a unique key to use to store state in - the dictionary. It is okay to call this function when no :term:`thread state` - is :term:`attached `. If this function returns - ``NULL``, no exception has been raised and the caller should assume no - thread state is attached. - - -.. c:function:: int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) - - Asynchronously raise an exception in a thread. The *id* argument is the thread - id of the target thread; *exc* is the exception object to be raised. This - function does not steal any references to *exc*. To prevent naive misuse, you - must write your own C extension to call this. Must be called with an :term:`attached thread state`. - Returns the number of thread states modified; this is normally one, but will be - zero if the thread id isn't found. If *exc* is ``NULL``, the pending - exception (if any) for the thread is cleared. This raises no exceptions. - - .. versionchanged:: 3.7 - The type of the *id* parameter changed from :c:expr:`long` to - :c:expr:`unsigned long`. - -.. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) - - :term:`Attach ` *tstate* to the current thread, - which must not be ``NULL`` or already :term:`attached `. - - The calling thread must not already have an :term:`attached thread state`. - - .. note:: - Calling this function from a thread when the runtime is finalizing will - hang the thread until the program exits, even if the thread was not - created by Python. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.8 - Updated to be consistent with :c:func:`PyEval_RestoreThread`, - :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, - and terminate the current thread if called while the interpreter is finalizing. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - - :c:func:`PyEval_RestoreThread` is a higher-level function which is always - available (even when threads have not been initialized). - - -.. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) - - Detach the :term:`attached thread state`. - The *tstate* argument, which must not be ``NULL``, is only used to check - that it represents the :term:`attached thread state` --- if it isn't, a fatal error is - reported. - - :c:func:`PyEval_SaveThread` is a higher-level function which is always - available (even when threads have not been initialized). - - -.. _sub-interpreter-support: - -Sub-interpreter support -======================= - -While in most uses, you will only embed a single Python interpreter, there -are cases where you need to create several independent interpreters in the -same process and perhaps even in the same thread. Sub-interpreters allow -you to do that. - -The "main" interpreter is the first one created when the runtime initializes. -It is usually the only Python interpreter in a process. Unlike sub-interpreters, -the main interpreter has unique process-global responsibilities like signal -handling. It is also responsible for execution during runtime initialization and -is usually the active interpreter during runtime finalization. The -:c:func:`PyInterpreterState_Main` function returns a pointer to its state. - -You can switch between sub-interpreters using the :c:func:`PyThreadState_Swap` -function. You can create and destroy them using the following functions: - - -.. c:type:: PyInterpreterConfig - - Structure containing most parameters to configure a sub-interpreter. - Its values are used only in :c:func:`Py_NewInterpreterFromConfig` and - never modified by the runtime. - - .. versionadded:: 3.12 - - Structure fields: - - .. c:member:: int use_main_obmalloc - - If this is ``0`` then the sub-interpreter will use its own - "object" allocator state. - Otherwise it will use (share) the main interpreter's. - - If this is ``0`` then - :c:member:`~PyInterpreterConfig.check_multi_interp_extensions` - must be ``1`` (non-zero). - If this is ``1`` then :c:member:`~PyInterpreterConfig.gil` - must not be :c:macro:`PyInterpreterConfig_OWN_GIL`. - - .. c:member:: int allow_fork - - If this is ``0`` then the runtime will not support forking the - process in any thread where the sub-interpreter is currently active. - Otherwise fork is unrestricted. - - Note that the :mod:`subprocess` module still works - when fork is disallowed. - - .. c:member:: int allow_exec - - If this is ``0`` then the runtime will not support replacing the - current process via exec (e.g. :func:`os.execv`) in any thread - where the sub-interpreter is currently active. - Otherwise exec is unrestricted. - - Note that the :mod:`subprocess` module still works - when exec is disallowed. - - .. c:member:: int allow_threads - - If this is ``0`` then the sub-interpreter's :mod:`threading` module - won't create threads. - Otherwise threads are allowed. - - .. c:member:: int allow_daemon_threads - - If this is ``0`` then the sub-interpreter's :mod:`threading` module - won't create daemon threads. - Otherwise daemon threads are allowed (as long as - :c:member:`~PyInterpreterConfig.allow_threads` is non-zero). - - .. c:member:: int check_multi_interp_extensions - - If this is ``0`` then all extension modules may be imported, - including legacy (single-phase init) modules, - in any thread where the sub-interpreter is currently active. - Otherwise only multi-phase init extension modules - (see :pep:`489`) may be imported. - (Also see :c:macro:`Py_mod_multiple_interpreters`.) - - This must be ``1`` (non-zero) if - :c:member:`~PyInterpreterConfig.use_main_obmalloc` is ``0``. - - .. c:member:: int gil - - This determines the operation of the GIL for the sub-interpreter. - It may be one of the following: - - .. c:namespace:: NULL - - .. c:macro:: PyInterpreterConfig_DEFAULT_GIL - - Use the default selection (:c:macro:`PyInterpreterConfig_SHARED_GIL`). - - .. c:macro:: PyInterpreterConfig_SHARED_GIL - - Use (share) the main interpreter's GIL. - - .. c:macro:: PyInterpreterConfig_OWN_GIL - - Use the sub-interpreter's own GIL. - - If this is :c:macro:`PyInterpreterConfig_OWN_GIL` then - :c:member:`PyInterpreterConfig.use_main_obmalloc` must be ``0``. - - -.. c:function:: PyStatus Py_NewInterpreterFromConfig(PyThreadState **tstate_p, const PyInterpreterConfig *config) - - .. index:: - pair: module; builtins - pair: module; __main__ - pair: module; sys - single: stdout (in module sys) - single: stderr (in module sys) - single: stdin (in module sys) - - Create a new sub-interpreter. This is an (almost) totally separate environment - for the execution of Python code. In particular, the new interpreter has - separate, independent versions of all imported modules, including the - fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. The - table of loaded modules (``sys.modules``) and the module search path - (``sys.path``) are also separate. The new environment has no ``sys.argv`` - variable. It has new standard I/O stream file objects ``sys.stdin``, - ``sys.stdout`` and ``sys.stderr`` (however these refer to the same underlying - file descriptors). - - The given *config* controls the options with which the interpreter - is initialized. - - Upon success, *tstate_p* will be set to the first :term:`thread state` - created in the new sub-interpreter. This thread state is - :term:`attached `. - Note that no actual thread is created; see the discussion of thread states - below. If creation of the new interpreter is unsuccessful, - *tstate_p* is set to ``NULL``; - no exception is set since the exception state is stored in the - :term:`attached thread state`, which might not exist. - - Like all other Python/C API functions, an :term:`attached thread state` - must be present before calling this function, but it might be detached upon - returning. On success, the returned thread state will be :term:`attached `. - If the sub-interpreter is created with its own :term:`GIL` then the - :term:`attached thread state` of the calling interpreter will be detached. - When the function returns, the new interpreter's :term:`thread state` - will be :term:`attached ` to the current thread and - the previous interpreter's :term:`attached thread state` will remain detached. - - .. versionadded:: 3.12 - - Sub-interpreters are most effective when isolated from each other, - with certain functionality restricted:: - - PyInterpreterConfig config = { - .use_main_obmalloc = 0, - .allow_fork = 0, - .allow_exec = 0, - .allow_threads = 1, - .allow_daemon_threads = 0, - .check_multi_interp_extensions = 1, - .gil = PyInterpreterConfig_OWN_GIL, - }; - PyThreadState *tstate = NULL; - PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } - - Note that the config is used only briefly and does not get modified. - During initialization the config's values are converted into various - :c:type:`PyInterpreterState` values. A read-only copy of the config - may be stored internally on the :c:type:`PyInterpreterState`. - - .. index:: - single: Py_FinalizeEx (C function) - single: Py_Initialize (C function) - - Extension modules are shared between (sub-)interpreters as follows: - - * For modules using multi-phase initialization, - e.g. :c:func:`PyModule_FromDefAndSpec`, a separate module object is - created and initialized for each interpreter. - Only C-level static and global variables are shared between these - module objects. - - * For modules using single-phase initialization, - e.g. :c:func:`PyModule_Create`, the first time a particular extension - is imported, it is initialized normally, and a (shallow) copy of its - module's dictionary is squirreled away. - When the same extension is imported by another (sub-)interpreter, a new - module is initialized and filled with the contents of this copy; the - extension's ``init`` function is not called. - Objects in the module's dictionary thus end up shared across - (sub-)interpreters, which might cause unwanted behavior (see - `Bugs and caveats`_ below). - - Note that this is different from what happens when an extension is - imported after the interpreter has been completely re-initialized by - calling :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that - case, the extension's ``initmodule`` function *is* called again. - As with multi-phase initialization, this means that only C-level static - and global variables are shared between these modules. - - .. index:: single: close (in module os) - - -.. c:function:: PyThreadState* Py_NewInterpreter(void) - - .. index:: - pair: module; builtins - pair: module; __main__ - pair: module; sys - single: stdout (in module sys) - single: stderr (in module sys) - single: stdin (in module sys) - - Create a new sub-interpreter. This is essentially just a wrapper - around :c:func:`Py_NewInterpreterFromConfig` with a config that - preserves the existing behavior. The result is an unisolated - sub-interpreter that shares the main interpreter's GIL, allows - fork/exec, allows daemon threads, and allows single-phase init - modules. - - -.. c:function:: void Py_EndInterpreter(PyThreadState *tstate) - - .. index:: single: Py_FinalizeEx (C function) - - Destroy the (sub-)interpreter represented by the given :term:`thread state`. - The given thread state must be :term:`attached `. - When the call returns, there will be no :term:`attached thread state`. - All thread states associated with this interpreter are destroyed. - - :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that - haven't been explicitly destroyed at that point. - - -.. _per-interpreter-gil: - -A Per-Interpreter GIL ---------------------- - -Using :c:func:`Py_NewInterpreterFromConfig` you can create -a sub-interpreter that is completely isolated from other interpreters, -including having its own GIL. The most important benefit of this -isolation is that such an interpreter can execute Python code without -being blocked by other interpreters or blocking any others. Thus a -single Python process can truly take advantage of multiple CPU cores -when running Python code. The isolation also encourages a different -approach to concurrency than that of just using threads. -(See :pep:`554` and :pep:`684`.) - -Using an isolated interpreter requires vigilance in preserving that -isolation. That especially means not sharing any objects or mutable -state without guarantees about thread-safety. Even objects that are -otherwise immutable (e.g. ``None``, ``(1, 5)``) can't normally be shared -because of the refcount. One simple but less-efficient approach around -this is to use a global lock around all use of some state (or object). -Alternately, effectively immutable objects (like integers or strings) -can be made safe in spite of their refcounts by making them :term:`immortal`. -In fact, this has been done for the builtin singletons, small integers, -and a number of other builtin objects. - -If you preserve isolation then you will have access to proper multi-core -computing without the complications that come with free-threading. -Failure to preserve isolation will expose you to the full consequences -of free-threading, including races and hard-to-debug crashes. - -Aside from that, one of the main challenges of using multiple isolated -interpreters is how to communicate between them safely (not break -isolation) and efficiently. The runtime and stdlib do not provide -any standard approach to this yet. A future stdlib module would help -mitigate the effort of preserving isolation and expose effective tools -for communicating (and sharing) data between interpreters. - -.. versionadded:: 3.12 - - -Bugs and caveats ----------------- - -Because sub-interpreters (and the main interpreter) are part of the same -process, the insulation between them isn't perfect --- for example, using -low-level file operations like :func:`os.close` they can -(accidentally or maliciously) affect each other's open files. Because of the -way extensions are shared between (sub-)interpreters, some extensions may not -work properly; this is especially likely when using single-phase initialization -or (static) global variables. -It is possible to insert objects created in one sub-interpreter into -a namespace of another (sub-)interpreter; this should be avoided if possible. - -Special care should be taken to avoid sharing user-defined functions, -methods, instances or classes between sub-interpreters, since import -operations executed by such objects may affect the wrong (sub-)interpreter's -dictionary of loaded modules. It is equally important to avoid sharing -objects from which the above are reachable. - -Also note that combining this functionality with ``PyGILState_*`` APIs -is delicate, because these APIs assume a bijection between Python thread states -and OS-level threads, an assumption broken by the presence of sub-interpreters. -It is highly recommended that you don't switch sub-interpreters between a pair -of matching :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` calls. -Furthermore, extensions (such as :mod:`ctypes`) using these APIs to allow calling -of Python code from non-Python created threads will probably be broken when using -sub-interpreters. - - -Asynchronous Notifications -========================== - -A mechanism is provided to make asynchronous notifications to the main -interpreter thread. These notifications take the form of a function -pointer and a void pointer argument. - - -.. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) - - Schedule a function to be called from the main interpreter thread. On - success, ``0`` is returned and *func* is queued for being called in the - main thread. On failure, ``-1`` is returned without setting any exception. - - When successfully queued, *func* will be *eventually* called from the - main interpreter thread with the argument *arg*. It will be called - asynchronously with respect to normally running Python code, but with - both these conditions met: - - * on a :term:`bytecode` boundary; - * with the main thread holding an :term:`attached thread state` - (*func* can therefore use the full C API). - - *func* must return ``0`` on success, or ``-1`` on failure with an exception - set. *func* won't be interrupted to perform another asynchronous - notification recursively, but it can still be interrupted to switch - threads if the :term:`thread state ` is detached. - - This function doesn't need an :term:`attached thread state`. However, to call this - function in a subinterpreter, the caller must have an :term:`attached thread state`. - Otherwise, the function *func* can be scheduled to be called from the wrong interpreter. - - .. warning:: - This is a low-level function, only useful for very special cases. - There is no guarantee that *func* will be called as quick as - possible. If the main thread is busy executing a system call, - *func* won't be called before the system call returns. This - function is generally **not** suitable for calling Python code from - arbitrary C threads. Instead, use the :ref:`PyGILState API`. - - .. versionadded:: 3.1 - - .. versionchanged:: 3.9 - If this function is called in a subinterpreter, the function *func* is - now scheduled to be called from the subinterpreter, rather than being - called from the main interpreter. Each subinterpreter now has its own - list of scheduled calls. - -.. _profiling: - -Profiling and Tracing -===================== - -.. sectionauthor:: Fred L. Drake, Jr. - - -The Python interpreter provides some low-level support for attaching profiling -and execution tracing facilities. These are used for profiling, debugging, and -coverage analysis tools. - -This C interface allows the profiling or tracing code to avoid the overhead of -calling through Python-level callable objects, making a direct C function call -instead. The essential attributes of the facility have not changed; the -interface allows trace functions to be installed per-thread, and the basic -events reported to the trace function are the same as had been reported to the -Python-level trace functions in previous versions. - - -.. c:type:: int (*Py_tracefunc)(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) - - The type of the trace function registered using :c:func:`PyEval_SetProfile` and - :c:func:`PyEval_SetTrace`. The first parameter is the object passed to the - registration function as *obj*, *frame* is the frame object to which the event - pertains, *what* is one of the constants :c:data:`PyTrace_CALL`, - :c:data:`PyTrace_EXCEPTION`, :c:data:`PyTrace_LINE`, :c:data:`PyTrace_RETURN`, - :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION`, :c:data:`PyTrace_C_RETURN`, - or :c:data:`PyTrace_OPCODE`, and *arg* depends on the value of *what*: - - +-------------------------------+----------------------------------------+ - | Value of *what* | Meaning of *arg* | - +===============================+========================================+ - | :c:data:`PyTrace_CALL` | Always :c:data:`Py_None`. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_EXCEPTION` | Exception information as returned by | - | | :func:`sys.exc_info`. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_LINE` | Always :c:data:`Py_None`. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_RETURN` | Value being returned to the caller, | - | | or ``NULL`` if caused by an exception. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_C_CALL` | Function object being called. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_C_EXCEPTION` | Function object being called. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_C_RETURN` | Function object being called. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_OPCODE` | Always :c:data:`Py_None`. | - +-------------------------------+----------------------------------------+ - -.. c:var:: int PyTrace_CALL - - The value of the *what* parameter to a :c:type:`Py_tracefunc` function when a new - call to a function or method is being reported, or a new entry into a generator. - Note that the creation of the iterator for a generator function is not reported - as there is no control transfer to the Python bytecode in the corresponding - frame. - - -.. c:var:: int PyTrace_EXCEPTION - - The value of the *what* parameter to a :c:type:`Py_tracefunc` function when an - exception has been raised. The callback function is called with this value for - *what* when after any bytecode is processed after which the exception becomes - set within the frame being executed. The effect of this is that as exception - propagation causes the Python stack to unwind, the callback is called upon - return to each frame as the exception propagates. Only trace functions receives - these events; they are not needed by the profiler. - - -.. c:var:: int PyTrace_LINE - - The value passed as the *what* parameter to a :c:type:`Py_tracefunc` function - (but not a profiling function) when a line-number event is being reported. - It may be disabled for a frame by setting :attr:`~frame.f_trace_lines` to - *0* on that frame. - - -.. c:var:: int PyTrace_RETURN - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a - call is about to return. - - -.. c:var:: int PyTrace_C_CALL - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function is about to be called. - - -.. c:var:: int PyTrace_C_EXCEPTION - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function has raised an exception. - - -.. c:var:: int PyTrace_C_RETURN - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function has returned. - - -.. c:var:: int PyTrace_OPCODE - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions (but not - profiling functions) when a new opcode is about to be executed. This event is - not emitted by default: it must be explicitly requested by setting - :attr:`~frame.f_trace_opcodes` to *1* on the frame. - - -.. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) - - Set the profiler function to *func*. The *obj* parameter is passed to the - function as its first parameter, and may be any Python object, or ``NULL``. If - the profile function needs to maintain state, using a different value for *obj* - for each thread provides a convenient and thread-safe place to store it. The - profile function is called for all monitored events except :c:data:`PyTrace_LINE` - :c:data:`PyTrace_OPCODE` and :c:data:`PyTrace_EXCEPTION`. - - See also the :func:`sys.setprofile` function. - - The caller must have an :term:`attached thread state`. - -.. c:function:: void PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *obj) - - Like :c:func:`PyEval_SetProfile` but sets the profile function in all running threads - belonging to the current interpreter instead of the setting it only on the current thread. - - The caller must have an :term:`attached thread state`. - - As :c:func:`PyEval_SetProfile`, this function ignores any exceptions raised while - setting the profile functions in all threads. - -.. versionadded:: 3.12 - - -.. c:function:: void PyEval_SetTrace(Py_tracefunc func, PyObject *obj) - - Set the tracing function to *func*. This is similar to - :c:func:`PyEval_SetProfile`, except the tracing function does receive line-number - events and per-opcode events, but does not receive any event related to C function - objects being called. Any trace function registered using :c:func:`PyEval_SetTrace` - will not receive :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION` or - :c:data:`PyTrace_C_RETURN` as a value for the *what* parameter. - - See also the :func:`sys.settrace` function. - - The caller must have an :term:`attached thread state`. - -.. c:function:: void PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *obj) - - Like :c:func:`PyEval_SetTrace` but sets the tracing function in all running threads - belonging to the current interpreter instead of the setting it only on the current thread. - - The caller must have an :term:`attached thread state`. - - As :c:func:`PyEval_SetTrace`, this function ignores any exceptions raised while - setting the trace functions in all threads. - -.. versionadded:: 3.12 - -Reference tracing -================= - -.. versionadded:: 3.13 - -.. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data) - - The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`. - The first parameter is a Python object that has been just created (when **event** - is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event** - is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer - that was provided when :c:func:`PyRefTracer_SetTracer` was called. - -.. versionadded:: 3.13 - -.. c:var:: int PyRefTracer_CREATE - - The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python - object has been created. - -.. c:var:: int PyRefTracer_DESTROY - - The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python - object has been destroyed. - -.. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) - - Register a reference tracer function. The function will be called when a new - Python has been created or when an object is going to be destroyed. If - **data** is provided it must be an opaque pointer that will be provided when - the tracer function is called. Return ``0`` on success. Set an exception and - return ``-1`` on error. - - Not that tracer functions **must not** create Python objects inside or - otherwise the call will be re-entrant. The tracer also **must not** clear - any existing exception or set an exception. A :term:`thread state` will be active - every time the tracer function is called. - - There must be an :term:`attached thread state` when calling this function. - -.. versionadded:: 3.13 - -.. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data) - - Get the registered reference tracer function and the value of the opaque data - pointer that was registered when :c:func:`PyRefTracer_SetTracer` was called. - If no tracer was registered this function will return NULL and will set the - **data** pointer to NULL. - - There must be an :term:`attached thread state` when calling this function. - -.. versionadded:: 3.13 - -.. _advanced-debugging: - -Advanced Debugger Support -========================= - -.. sectionauthor:: Fred L. Drake, Jr. - - -These functions are only intended to be used by advanced debugging tools. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Head() - - Return the interpreter state object at the head of the list of all such objects. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Main() - - Return the main interpreter state object. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Next(PyInterpreterState *interp) - - Return the next interpreter state object after *interp* from the list of all - such objects. - - -.. c:function:: PyThreadState * PyInterpreterState_ThreadHead(PyInterpreterState *interp) - - Return the pointer to the first :c:type:`PyThreadState` object in the list of - threads associated with the interpreter *interp*. - - -.. c:function:: PyThreadState* PyThreadState_Next(PyThreadState *tstate) - - Return the next thread state object after *tstate* from the list of all such - objects belonging to the same :c:type:`PyInterpreterState` object. - - -.. _thread-local-storage: - -Thread Local Storage Support -============================ - -.. sectionauthor:: Masayuki Yamamoto - -The Python interpreter provides low-level support for thread-local storage -(TLS) which wraps the underlying native TLS implementation to support the -Python-level thread local storage API (:class:`threading.local`). The -CPython C level APIs are similar to those offered by pthreads and Windows: -use a thread key and functions to associate a :c:expr:`void*` value per -thread. - -A :term:`thread state` does *not* need to be :term:`attached ` -when calling these functions; they suppl their own locking. - -Note that :file:`Python.h` does not include the declaration of the TLS APIs, -you need to include :file:`pythread.h` to use thread-local storage. - -.. note:: - None of these API functions handle memory management on behalf of the - :c:expr:`void*` values. You need to allocate and deallocate them yourself. - If the :c:expr:`void*` values happen to be :c:expr:`PyObject*`, these - functions don't do refcount operations on them either. - -.. _thread-specific-storage-api: - -Thread Specific Storage (TSS) API ---------------------------------- - -TSS API is introduced to supersede the use of the existing TLS API within the -CPython interpreter. This API uses a new type :c:type:`Py_tss_t` instead of -:c:expr:`int` to represent thread keys. - -.. versionadded:: 3.7 - -.. seealso:: "A New C-API for Thread-Local Storage in CPython" (:pep:`539`) - - -.. c:type:: Py_tss_t - - This data structure represents the state of a thread key, the definition of - which may depend on the underlying TLS implementation, and it has an - internal field representing the key's initialization state. There are no - public members in this structure. - - When :ref:`Py_LIMITED_API ` is not defined, static allocation of - this type by :c:macro:`Py_tss_NEEDS_INIT` is allowed. - - -.. c:macro:: Py_tss_NEEDS_INIT - - This macro expands to the initializer for :c:type:`Py_tss_t` variables. - Note that this macro won't be defined with :ref:`Py_LIMITED_API `. - - -Dynamic Allocation -~~~~~~~~~~~~~~~~~~ - -Dynamic allocation of the :c:type:`Py_tss_t`, required in extension modules -built with :ref:`Py_LIMITED_API `, where static allocation of this type -is not possible due to its implementation being opaque at build time. - - -.. c:function:: Py_tss_t* PyThread_tss_alloc() - - Return a value which is the same state as a value initialized with - :c:macro:`Py_tss_NEEDS_INIT`, or ``NULL`` in the case of dynamic allocation - failure. - - -.. c:function:: void PyThread_tss_free(Py_tss_t *key) - - Free the given *key* allocated by :c:func:`PyThread_tss_alloc`, after - first calling :c:func:`PyThread_tss_delete` to ensure any associated - thread locals have been unassigned. This is a no-op if the *key* - argument is ``NULL``. - - .. note:: - A freed key becomes a dangling pointer. You should reset the key to - ``NULL``. - - -Methods -~~~~~~~ - -The parameter *key* of these functions must not be ``NULL``. Moreover, the -behaviors of :c:func:`PyThread_tss_set` and :c:func:`PyThread_tss_get` are -undefined if the given :c:type:`Py_tss_t` has not been initialized by -:c:func:`PyThread_tss_create`. - - -.. c:function:: int PyThread_tss_is_created(Py_tss_t *key) - - Return a non-zero value if the given :c:type:`Py_tss_t` has been initialized - by :c:func:`PyThread_tss_create`. - - -.. c:function:: int PyThread_tss_create(Py_tss_t *key) - - Return a zero value on successful initialization of a TSS key. The behavior - is undefined if the value pointed to by the *key* argument is not - initialized by :c:macro:`Py_tss_NEEDS_INIT`. This function can be called - repeatedly on the same key -- calling it on an already initialized key is a - no-op and immediately returns success. - - -.. c:function:: void PyThread_tss_delete(Py_tss_t *key) - - Destroy a TSS key to forget the values associated with the key across all - threads, and change the key's initialization state to uninitialized. A - destroyed key is able to be initialized again by - :c:func:`PyThread_tss_create`. This function can be called repeatedly on - the same key -- calling it on an already destroyed key is a no-op. - - -.. c:function:: int PyThread_tss_set(Py_tss_t *key, void *value) - - Return a zero value to indicate successfully associating a :c:expr:`void*` - value with a TSS key in the current thread. Each thread has a distinct - mapping of the key to a :c:expr:`void*` value. - - -.. c:function:: void* PyThread_tss_get(Py_tss_t *key) - - Return the :c:expr:`void*` value associated with a TSS key in the current - thread. This returns ``NULL`` if no value is associated with the key in the - current thread. - - -.. _thread-local-storage-api: - -Thread Local Storage (TLS) API ------------------------------- - -.. deprecated:: 3.7 - This API is superseded by - :ref:`Thread Specific Storage (TSS) API `. - -.. note:: - This version of the API does not support platforms where the native TLS key - is defined in a way that cannot be safely cast to ``int``. On such platforms, - :c:func:`PyThread_create_key` will return immediately with a failure status, - and the other TLS functions will all be no-ops on such platforms. - -Due to the compatibility problem noted above, this version of the API should not -be used in new code. - -.. c:function:: int PyThread_create_key() -.. c:function:: void PyThread_delete_key(int key) -.. c:function:: int PyThread_set_key_value(int key, void *value) -.. c:function:: void* PyThread_get_key_value(int key) -.. c:function:: void PyThread_delete_key_value(int key) -.. c:function:: void PyThread_ReInitTLS() - -Synchronization Primitives -========================== - -The C-API provides a basic mutual exclusion lock. - -.. c:type:: PyMutex - - A mutual exclusion lock. The :c:type:`!PyMutex` should be initialized to - zero to represent the unlocked state. For example:: - - PyMutex mutex = {0}; - - Instances of :c:type:`!PyMutex` should not be copied or moved. Both the - contents and address of a :c:type:`!PyMutex` are meaningful, and it must - remain at a fixed, writable location in memory. - - .. note:: - - A :c:type:`!PyMutex` currently occupies one byte, but the size should be - considered unstable. The size may change in future Python releases - without a deprecation period. - - .. versionadded:: 3.13 - -.. c:function:: void PyMutex_Lock(PyMutex *m) - - Lock mutex *m*. If another thread has already locked it, the calling - thread will block until the mutex is unlocked. While blocked, the thread - will temporarily detach the :term:`thread state ` if one exists. - - .. versionadded:: 3.13 - -.. c:function:: void PyMutex_Unlock(PyMutex *m) - - Unlock mutex *m*. The mutex must be locked --- otherwise, the function will - issue a fatal error. - - .. versionadded:: 3.13 - -.. c:function:: int PyMutex_IsLocked(PyMutex *m) - - Returns non-zero if the mutex *m* is currently locked, zero otherwise. - - .. note:: - - This function is intended for use in assertions and debugging only and - should not be used to make concurrency control decisions, as the lock - state may change immediately after the check. - - .. versionadded:: 3.14 - -.. _python-critical-section-api: - -Python Critical Section API ---------------------------- - -The critical section API provides a deadlock avoidance layer on top of -per-object locks for :term:`free-threaded ` CPython. They are -intended to replace reliance on the :term:`global interpreter lock`, and are -no-ops in versions of Python with the global interpreter lock. - -Critical sections are intended to be used for custom types implemented -in C-API extensions. They should generally not be used with built-in types like -:class:`list` and :class:`dict` because their public C-APIs -already use critical sections internally, with the notable -exception of :c:func:`PyDict_Next`, which requires critical section -to be acquired externally. - -Critical sections avoid deadlocks by implicitly suspending active critical -sections, hence, they do not provide exclusive access such as provided by -traditional locks like :c:type:`PyMutex`. When a critical section is started, -the per-object lock for the object is acquired. If the code executed inside the -critical section calls C-API functions then it can suspend the critical section thereby -releasing the per-object lock, so other threads can acquire the per-object lock -for the same object. - -Variants that accept :c:type:`PyMutex` pointers rather than Python objects are also -available. Use these variants to start a critical section in a situation where -there is no :c:type:`PyObject` -- for example, when working with a C type that -does not extend or wrap :c:type:`PyObject` but still needs to call into the C -API in a manner that might lead to deadlocks. - -The functions and structs used by the macros are exposed for cases -where C macros are not available. They should only be used as in the -given macro expansions. Note that the sizes and contents of the structures may -change in future Python versions. - -.. note:: - - Operations that need to lock two objects at once must use - :c:macro:`Py_BEGIN_CRITICAL_SECTION2`. You *cannot* use nested critical - sections to lock more than one object at once, because the inner critical - section may suspend the outer critical sections. This API does not provide - a way to lock more than two objects at once. - -Example usage:: - - static PyObject * - set_field(MyObject *self, PyObject *value) - { - Py_BEGIN_CRITICAL_SECTION(self); - Py_SETREF(self->field, Py_XNewRef(value)); - Py_END_CRITICAL_SECTION(); - Py_RETURN_NONE; - } - -In the above example, :c:macro:`Py_SETREF` calls :c:macro:`Py_DECREF`, which -can call arbitrary code through an object's deallocation function. The critical -section API avoids potential deadlocks due to reentrancy and lock ordering -by allowing the runtime to temporarily suspend the critical section if the -code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. - -.. c:macro:: Py_BEGIN_CRITICAL_SECTION(op) - - Acquires the per-object lock for the object *op* and begins a - critical section. - - In the free-threaded build, this macro expands to:: - - { - PyCriticalSection _py_cs; - PyCriticalSection_Begin(&_py_cs, (PyObject*)(op)) - - In the default build, this macro expands to ``{``. - - .. versionadded:: 3.13 - -.. c:macro:: Py_BEGIN_CRITICAL_SECTION_MUTEX(m) - - Locks the mutex *m* and begins a critical section. - - In the free-threaded build, this macro expands to:: - - { - PyCriticalSection _py_cs; - PyCriticalSection_BeginMutex(&_py_cs, m) - - Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION`, there is no cast for - the argument of the macro - it must be a :c:type:`PyMutex` pointer. - - On the default build, this macro expands to ``{``. - - .. versionadded:: 3.14 - -.. c:macro:: Py_END_CRITICAL_SECTION() - - Ends the critical section and releases the per-object lock. - - In the free-threaded build, this macro expands to:: - - PyCriticalSection_End(&_py_cs); - } - - In the default build, this macro expands to ``}``. - - .. versionadded:: 3.13 - -.. c:macro:: Py_BEGIN_CRITICAL_SECTION2(a, b) - - Acquires the per-objects locks for the objects *a* and *b* and begins a - critical section. The locks are acquired in a consistent order (lowest - address first) to avoid lock ordering deadlocks. - - In the free-threaded build, this macro expands to:: - - { - PyCriticalSection2 _py_cs2; - PyCriticalSection2_Begin(&_py_cs2, (PyObject*)(a), (PyObject*)(b)) - - In the default build, this macro expands to ``{``. - - .. versionadded:: 3.13 - -.. c:macro:: Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) - - Locks the mutexes *m1* and *m2* and begins a critical section. - - In the free-threaded build, this macro expands to:: - - { - PyCriticalSection2 _py_cs2; - PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) - - Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION2`, there is no cast for - the arguments of the macro - they must be :c:type:`PyMutex` pointers. - - On the default build, this macro expands to ``{``. - - .. versionadded:: 3.14 - -.. c:macro:: Py_END_CRITICAL_SECTION2() - - Ends the critical section and releases the per-object locks. - - In the free-threaded build, this macro expands to:: - - PyCriticalSection2_End(&_py_cs2); - } - - In the default build, this macro expands to ``}``. - - .. versionadded:: 3.13 +- :ref:`initialization` +- :ref:`threads` +- :ref:`synchronization` +- :ref:`thread-local-storage` +- :ref:`sub-interpreter-support` +- :ref:`profiling` diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 4f0199838e1..209e48767cc 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -102,7 +102,7 @@ Error Handling * Set *\*err_msg* and return ``1`` if an error is set. * Set *\*err_msg* to ``NULL`` and return ``0`` otherwise. - An error message is an UTF-8 encoded string. + An error message is a UTF-8 encoded string. If *config* has an exit code, format the exit code as an error message. @@ -544,9 +544,9 @@ Configuration Options Visibility: -* Public: Can by get by :c:func:`PyConfig_Get` and set by +* Public: Can be retrieved by :c:func:`PyConfig_Get` and set by :c:func:`PyConfig_Set`. -* Read-only: Can by get by :c:func:`PyConfig_Get`, but cannot be set by +* Read-only: Can be retrieved by :c:func:`PyConfig_Get`, but cannot be set by :c:func:`PyConfig_Set`. @@ -1153,7 +1153,7 @@ PyConfig Most ``PyConfig`` methods :ref:`preinitialize Python ` if needed. In that case, the Python preinitialization configuration - (:c:type:`PyPreConfig`) in based on the :c:type:`PyConfig`. If configuration + (:c:type:`PyPreConfig`) is based on the :c:type:`PyConfig`. If configuration fields which are in common with :c:type:`PyPreConfig` are tuned, they must be set before calling a :c:type:`PyConfig` method: @@ -1278,6 +1278,11 @@ PyConfig Default: ``0``. + .. deprecated-removed:: 3.15 3.17 + + The :option:`-b` and :option:`!-bb` options will become no-op in 3.17. + :c:member:`~PyConfig.bytes_warning` member will be removed in 3.17. + .. c:member:: int warn_default_encoding If non-zero, emit a :exc:`EncodingWarning` warning when :class:`io.TextIOWrapper` @@ -1802,10 +1807,10 @@ PyConfig .. c:member:: wchar_t* run_presite - ``package.module`` path to module that should be imported before - ``site.py`` is run. + ``module`` or ``module:func`` entry point that should be executed before + the :mod:`site` module is imported. - Set by the :option:`-X presite=package.module <-X>` command-line + Set by the :option:`-X presite=module:func <-X>` command-line option and the :envvar:`PYTHON_PRESITE` environment variable. The command-line option takes precedence. @@ -2294,13 +2299,91 @@ Py_GetArgcArgv() See also :c:member:`PyConfig.orig_argv` member. -Delaying main module execution -============================== -In some embedding use cases, it may be desirable to separate interpreter initialization -from the execution of the main module. +Multi-Phase Initialization Private Provisional API +================================================== -This separation can be achieved by setting ``PyConfig.run_command`` to the empty -string during initialization (to prevent the interpreter from dropping into the -interactive prompt), and then subsequently executing the desired main module -code using ``__main__.__dict__`` as the global namespace. +This section is a private provisional API introducing multi-phase +initialization, the core feature of :pep:`432`: + +* "Core" initialization phase, "bare minimum Python": + + * Builtin types; + * Builtin exceptions; + * Builtin and frozen modules; + * The :mod:`sys` module is only partially initialized + (ex: :data:`sys.path` doesn't exist yet). + +* "Main" initialization phase, Python is fully initialized: + + * Install and configure :mod:`importlib`; + * Apply the :ref:`Path Configuration `; + * Install signal handlers; + * Finish :mod:`sys` module initialization (ex: create :data:`sys.stdout` + and :data:`sys.path`); + * Enable optional features like :mod:`faulthandler` and :mod:`tracemalloc`; + * Import the :mod:`site` module; + * etc. + +Private provisional API: + +.. c:member:: int PyConfig._init_main + + If set to ``0``, :c:func:`Py_InitializeFromConfig` stops at the "Core" + initialization phase. + +.. c:function:: PyStatus _Py_InitializeMain(void) + + Move to the "Main" initialization phase, finish the Python initialization. + +No module is imported during the "Core" phase and the ``importlib`` module is +not configured: the :ref:`Path Configuration ` is only +applied during the "Main" phase. It may allow to customize Python in Python to +override or tune the :ref:`Path Configuration `, maybe +install a custom :data:`sys.meta_path` importer or an import hook, etc. + +It may become possible to calculate the :ref:`Path Configuration +` in Python, after the Core phase and before the Main phase, +which is one of the :pep:`432` motivation. + +The "Core" phase is not properly defined: what should be and what should +not be available at this phase is not specified yet. The API is marked +as private and provisional: the API can be modified or even be removed +anytime until a proper public API is designed. + +Example running Python code between "Core" and "Main" initialization +phases:: + + void init_python(void) + { + PyStatus status; + + PyConfig config; + PyConfig_InitPythonConfig(&config); + config._init_main = 0; + + /* ... customize 'config' configuration ... */ + + status = Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + + /* Use sys.stderr because sys.stdout is only created + by _Py_InitializeMain() */ + int res = PyRun_SimpleString( + "import sys; " + "print('Run Python code before _Py_InitializeMain', " + "file=sys.stderr)"); + if (res < 0) { + exit(1); + } + + /* ... put more configuration code here ... */ + + status = _Py_InitializeMain(); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + } diff --git a/Doc/c-api/interp-lifecycle.rst b/Doc/c-api/interp-lifecycle.rst new file mode 100644 index 00000000000..186ab4370bc --- /dev/null +++ b/Doc/c-api/interp-lifecycle.rst @@ -0,0 +1,802 @@ +.. highlight:: c + +.. _initialization: + +Interpreter initialization and finalization +=========================================== + +See :ref:`Python Initialization Configuration ` for details +on how to configure the interpreter prior to initialization. + +.. _pre-init-safe: + +Before Python initialization +---------------------------- + +In an application embedding Python, the :c:func:`Py_Initialize` function must +be called before using any other Python/C API functions; with the exception of +a few functions and the :ref:`global configuration variables +`. + +The following functions can be safely called before Python is initialized: + +* Functions that initialize the interpreter: + + * :c:func:`Py_Initialize` + * :c:func:`Py_InitializeEx` + * :c:func:`Py_InitializeFromConfig` + * :c:func:`Py_BytesMain` + * :c:func:`Py_Main` + * the runtime pre-initialization functions covered in :ref:`init-config` + +* Configuration functions: + + * :c:func:`PyImport_AppendInittab` + * :c:func:`PyImport_ExtendInittab` + * :c:func:`!PyInitFrozenExtensions` + * :c:func:`PyMem_SetAllocator` + * :c:func:`PyMem_SetupDebugHooks` + * :c:func:`PyObject_SetArenaAllocator` + * :c:func:`Py_SetProgramName` + * :c:func:`Py_SetPythonHome` + * the configuration functions covered in :ref:`init-config` + +* Informative functions: + + * :c:func:`Py_IsInitialized` + * :c:func:`PyMem_GetAllocator` + * :c:func:`PyObject_GetArenaAllocator` + * :c:func:`Py_GetBuildInfo` + * :c:func:`Py_GetCompiler` + * :c:func:`Py_GetCopyright` + * :c:func:`Py_GetPlatform` + * :c:func:`Py_GetVersion` + * :c:func:`Py_IsInitialized` + +* Utilities: + + * :c:func:`Py_DecodeLocale` + * the status reporting and utility functions covered in :ref:`init-config` + +* Memory allocators: + + * :c:func:`PyMem_RawMalloc` + * :c:func:`PyMem_RawRealloc` + * :c:func:`PyMem_RawCalloc` + * :c:func:`PyMem_RawFree` + +* Synchronization: + + * :c:func:`PyMutex_Lock` + * :c:func:`PyMutex_Unlock` + +.. note:: + + Despite their apparent similarity to some of the functions listed above, + the following functions **should not be called** before the interpreter has + been initialized: :c:func:`Py_EncodeLocale`, :c:func:`PyEval_InitThreads`, and + :c:func:`Py_RunMain`. + + +.. _global-conf-vars: + +Global configuration variables +------------------------------ + +Python has variables for the global configuration to control different features +and options. By default, these flags are controlled by :ref:`command line +options `. + +When a flag is set by an option, the value of the flag is the number of times +that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag` +to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. + + +.. c:var:: int Py_BytesWarningFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.bytes_warning` should be used instead, see :ref:`Python + Initialization Configuration `. + + Issue a warning when comparing :class:`bytes` or :class:`bytearray` with + :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater + or equal to ``2``. + + Set by the :option:`-b` option. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_DebugFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.parser_debug` should be used instead, see :ref:`Python + Initialization Configuration `. + + Turn on parser debugging output (for expert only, depending on compilation + options). + + Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment + variable. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_DontWriteBytecodeFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.write_bytecode` should be used instead, see :ref:`Python + Initialization Configuration `. + + If set to non-zero, Python won't try to write ``.pyc`` files on the + import of source modules. + + Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` + environment variable. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_FrozenFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.pathconfig_warnings` should be used instead, see + :ref:`Python Initialization Configuration `. + + Private flag used by ``_freeze_module`` and ``frozenmain`` programs. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_HashRandomizationFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.hash_seed` and :c:member:`PyConfig.use_hash_seed` should + be used instead, see :ref:`Python Initialization Configuration + `. + + Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to + a non-empty string. + + If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment + variable to initialize the secret hash seed. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_IgnoreEnvironmentFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.use_environment` should be used instead, see + :ref:`Python Initialization Configuration `. + + Ignore all :envvar:`!PYTHON*` environment variables, e.g. + :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. + + Set by the :option:`-E` and :option:`-I` options. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_InspectFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.inspect` should be used instead, see + :ref:`Python Initialization Configuration `. + + When a script is passed as first argument or the :option:`-c` option is used, + enter interactive mode after executing the script or the command, even when + :data:`sys.stdin` does not appear to be a terminal. + + Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment + variable. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_InteractiveFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.interactive` should be used instead, see + :ref:`Python Initialization Configuration `. + + Set by the :option:`-i` option. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_IsolatedFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.isolated` should be used instead, see + :ref:`Python Initialization Configuration `. + + Run Python in isolated mode. In isolated mode :data:`sys.path` contains + neither the script's directory nor the user's site-packages directory. + + Set by the :option:`-I` option. + + .. versionadded:: 3.4 + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_LegacyWindowsFSEncodingFlag + + This API is kept for backward compatibility: setting + :c:member:`PyPreConfig.legacy_windows_fs_encoding` should be used instead, see + :ref:`Python Initialization Configuration `. + + If the flag is non-zero, use the ``mbcs`` encoding with ``replace`` error + handler, instead of the UTF-8 encoding with ``surrogatepass`` error handler, + for the :term:`filesystem encoding and error handler`. + + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment + variable is set to a non-empty string. + + See :pep:`529` for more details. + + .. availability:: Windows. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_LegacyWindowsStdioFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.legacy_windows_stdio` should be used instead, see + :ref:`Python Initialization Configuration `. + + If the flag is non-zero, use :class:`io.FileIO` instead of + :class:`!io._WindowsConsoleIO` for :mod:`sys` standard streams. + + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment + variable is set to a non-empty string. + + See :pep:`528` for more details. + + .. availability:: Windows. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_NoSiteFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.site_import` should be used instead, see + :ref:`Python Initialization Configuration `. + + Disable the import of the module :mod:`site` and the site-dependent + manipulations of :data:`sys.path` that it entails. Also disable these + manipulations if :mod:`site` is explicitly imported later (call + :func:`site.main` if you want them to be triggered). + + Set by the :option:`-S` option. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_NoUserSiteDirectory + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.user_site_directory` should be used instead, see + :ref:`Python Initialization Configuration `. + + Don't add the :data:`user site-packages directory ` to + :data:`sys.path`. + + Set by the :option:`-s` and :option:`-I` options, and the + :envvar:`PYTHONNOUSERSITE` environment variable. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_OptimizeFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.optimization_level` should be used instead, see + :ref:`Python Initialization Configuration `. + + Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment + variable. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_QuietFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.quiet` should be used instead, see :ref:`Python + Initialization Configuration `. + + Don't display the copyright and version messages even in interactive mode. + + Set by the :option:`-q` option. + + .. versionadded:: 3.2 + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_UnbufferedStdioFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.buffered_stdio` should be used instead, see :ref:`Python + Initialization Configuration `. + + Force the stdout and stderr streams to be unbuffered. + + Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` + environment variable. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_VerboseFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.verbose` should be used instead, see :ref:`Python + Initialization Configuration `. + + Print a message each time a module is initialized, showing the place + (filename or built-in module) from which it is loaded. If greater or equal + to ``2``, print a message for each file that is checked for when + searching for a module. Also provides information on module cleanup at exit. + + Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment + variable. + + .. deprecated-removed:: 3.12 3.15 + + +Initializing and finalizing the interpreter +------------------------------------------- + +.. c:function:: void Py_Initialize() + + .. index:: + single: PyEval_InitThreads() + single: modules (in module sys) + single: path (in module sys) + pair: module; builtins + pair: module; __main__ + pair: module; sys + triple: module; search; path + single: Py_FinalizeEx (C function) + + Initialize the Python interpreter. In an application embedding Python, + this should be called before using any other Python/C API functions; see + :ref:`Before Python Initialization ` for the few exceptions. + + This initializes the table of loaded modules (``sys.modules``), and creates + the fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. + It also initializes the module search path (``sys.path``). It does not set + ``sys.argv``; use the :ref:`Python Initialization Configuration ` + API for that. This is a no-op when called for a second time (without calling + :c:func:`Py_FinalizeEx` first). There is no return value; it is a fatal + error if the initialization fails. + + Use :c:func:`Py_InitializeFromConfig` to customize the + :ref:`Python Initialization Configuration `. + + .. note:: + On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, + which will also affect non-Python uses of the console using the C Runtime. + + +.. c:function:: void Py_InitializeEx(int initsigs) + + This function works like :c:func:`Py_Initialize` if *initsigs* is ``1``. If + *initsigs* is ``0``, it skips initialization registration of signal handlers, + which may be useful when CPython is embedded as part of a larger application. + + Use :c:func:`Py_InitializeFromConfig` to customize the + :ref:`Python Initialization Configuration `. + + +.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) + + Initialize Python from *config* configuration, as described in + :ref:`init-from-config`. + + See the :ref:`init-config` section for details on pre-initializing the + interpreter, populating the runtime configuration structure, and querying + the returned status structure. + + +.. c:function:: int Py_IsInitialized() + + Return true (nonzero) when the Python interpreter has been initialized, false + (zero) if not. After :c:func:`Py_FinalizeEx` is called, this returns false until + :c:func:`Py_Initialize` is called again. + + .. versionchanged:: next + This function no longer returns true until initialization has fully + completed, including import of the :mod:`site` module. Previously it + could return true while :c:func:`Py_Initialize` was still running. + + +.. c:function:: int Py_IsFinalizing() + + Return true (non-zero) if the main Python interpreter is + :term:`shutting down `. Return false (zero) otherwise. + + .. versionadded:: 3.13 + + +.. c:function:: int Py_FinalizeEx() + + Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of + Python/C API functions, and destroy all sub-interpreters (see + :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since + the last call to :c:func:`Py_Initialize`. This is a no-op when called for a second + time (without calling :c:func:`Py_Initialize` again first). + + Since this is the reverse of :c:func:`Py_Initialize`, it should be called + in the same thread with the same interpreter active. That means + the main thread and the main interpreter. + This should never be called while :c:func:`Py_RunMain` is running. + + Normally the return value is ``0``. + If there were errors during finalization (flushing buffered data), + ``-1`` is returned. + + Note that Python will do a best effort at freeing all memory allocated by the Python + interpreter. Therefore, any C-Extension should make sure to correctly clean up all + of the previously allocated PyObjects before using them in subsequent calls to + :c:func:`Py_Initialize`. Otherwise it could introduce vulnerabilities and incorrect + behavior. + + This function is provided for a number of reasons. An embedding application + might want to restart Python without having to restart the application itself. + An application that has loaded the Python interpreter from a dynamically + loadable library (or DLL) might want to free all memory allocated by Python + before unloading the DLL. During a hunt for memory leaks in an application a + developer might want to free all memory allocated by Python before exiting from + the application. + + **Bugs and caveats:** The destruction of modules and objects in modules is done + in random order; this may cause destructors (:meth:`~object.__del__` methods) to fail + when they depend on other objects (even functions) or modules. Dynamically + loaded extension modules loaded by Python are not unloaded. Small amounts of + memory allocated by the Python interpreter may not be freed (if you find a leak, + please report it). Memory tied up in circular references between objects is not + freed. Interned strings will all be deallocated regardless of their reference count. + Some memory allocated by extension modules may not be freed. Some extensions may not + work properly if their initialization routine is called more than once; this can + happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` + more than once. :c:func:`Py_FinalizeEx` must not be called recursively from + within itself. Therefore, it must not be called by any code that may be run + as part of the interpreter shutdown process, such as :py:mod:`atexit` + handlers, object finalizers, or any code that may be run while flushing the + stdout and stderr files. + + .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx + + .. versionadded:: 3.6 + + +.. c:function:: void Py_Finalize() + + This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that + disregards the return value. + + +.. c:function:: int Py_BytesMain(int argc, char **argv) + + Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings, + allowing the calling application to delegate the text decoding step to + the CPython runtime. + + .. versionadded:: 3.8 + + +.. c:function:: int Py_Main(int argc, wchar_t **argv) + + The main program for the standard interpreter, encapsulating a full + initialization/finalization cycle, as well as additional + behaviour to implement reading configurations settings from the environment + and command line, and then executing ``__main__`` in accordance with + :ref:`using-on-cmdline`. + + This is made available for programs which wish to support the full CPython + command line interface, rather than just embedding a Python runtime in a + larger application. + + The *argc* and *argv* parameters are similar to those which are passed to a + C program's :c:func:`main` function, except that the *argv* entries are first + converted to ``wchar_t`` using :c:func:`Py_DecodeLocale`. It is also + important to note that the argument list entries may be modified to point to + strings other than those passed in (however, the contents of the strings + pointed to by the argument list are not modified). + + The return value is ``2`` if the argument list does not represent a valid + Python command line, and otherwise the same as :c:func:`Py_RunMain`. + + In terms of the CPython runtime configuration APIs documented in the + :ref:`runtime configuration ` section (and without accounting + for error handling), ``Py_Main`` is approximately equivalent to:: + + PyConfig config; + PyConfig_InitPythonConfig(&config); + PyConfig_SetArgv(&config, argc, argv); + Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); + + Py_RunMain(); + + In normal usage, an embedding application will call this function + *instead* of calling :c:func:`Py_Initialize`, :c:func:`Py_InitializeEx` or + :c:func:`Py_InitializeFromConfig` directly, and all settings will be applied + as described elsewhere in this documentation. If this function is instead + called *after* a preceding runtime initialization API call, then exactly + which environmental and command line configuration settings will be updated + is version dependent (as it depends on which settings correctly support + being modified after they have already been set once when the runtime was + first initialized). + + +.. c:function:: int Py_RunMain(void) + + Executes the main module in a fully configured CPython runtime. + + Executes the command (:c:member:`PyConfig.run_command`), the script + (:c:member:`PyConfig.run_filename`) or the module + (:c:member:`PyConfig.run_module`) specified on the command line or in the + configuration. If none of these values are set, runs the interactive Python + prompt (REPL) using the ``__main__`` module's global namespace. + + If :c:member:`PyConfig.inspect` is not set (the default), the return value + will be ``0`` if the interpreter exits normally (that is, without raising + an exception), the exit status of an unhandled :exc:`SystemExit`, or ``1`` + for any other unhandled exception. + + If :c:member:`PyConfig.inspect` is set (such as when the :option:`-i` option + is used), rather than returning when the interpreter exits, execution will + instead resume in an interactive Python prompt (REPL) using the ``__main__`` + module's global namespace. If the interpreter exited with an exception, it + is immediately raised in the REPL session. The function return value is + then determined by the way the *REPL session* terminates: ``0``, ``1``, or + the status of a :exc:`SystemExit`, as specified above. + + This function always finalizes the Python interpreter before it returns. + + See :ref:`Python Configuration ` for an example of a + customized Python that always runs in isolated mode using + :c:func:`Py_RunMain`. + +.. c:function:: int PyUnstable_AtExit(PyInterpreterState *interp, void (*func)(void *), void *data) + + Register an :mod:`atexit` callback for the target interpreter *interp*. + This is similar to :c:func:`Py_AtExit`, but takes an explicit interpreter and + data pointer for the callback. + + There must be an :term:`attached thread state` for *interp*. + + .. versionadded:: 3.13 + + +.. _cautions-regarding-runtime-finalization: + +Cautions regarding runtime finalization +--------------------------------------- + +In the late stage of :term:`interpreter shutdown`, after attempting to wait for +non-daemon threads to exit (though this can be interrupted by +:class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime +is marked as *finalizing*: :c:func:`Py_IsFinalizing` and +:func:`sys.is_finalizing` return true. At this point, only the *finalization +thread* that initiated finalization (typically the main thread) is allowed to +acquire the :term:`GIL`. + +If any thread, other than the finalization thread, attempts to attach a :term:`thread state` +during finalization, either explicitly or +implicitly, the thread enters **a permanently blocked state** +where it remains until the program exits. In most cases this is harmless, but this can result +in deadlock if a later stage of finalization attempts to acquire a lock owned by the +blocked thread, or otherwise waits on the blocked thread. + +Gross? Yes. This prevents random crashes and/or unexpectedly skipped C++ +finalizations further up the call stack when such threads were forcibly exited +here in CPython 3.13 and earlier. The CPython runtime :term:`thread state` C APIs +have never had any error reporting or handling expectations at :term:`thread state` +attachment time that would've allowed for graceful exit from this situation. Changing that +would require new stable C APIs and rewriting the majority of C code in the +CPython ecosystem to use those with error handling. + + +Process-wide parameters +----------------------- + +.. c:function:: void Py_SetProgramName(const wchar_t *name) + + .. index:: + single: Py_Initialize() + single: main() + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.program_name` should be used instead, see :ref:`Python + Initialization Configuration `. + + This function should be called before :c:func:`Py_Initialize` is called for + the first time, if it is called at all. It tells the interpreter the value + of the ``argv[0]`` argument to the :c:func:`main` function of the program + (converted to wide characters). + This is used by some other functions below to find + the Python run-time libraries relative to the interpreter executable. The + default value is ``'python'``. The argument should point to a + zero-terminated wide character string in static storage whose contents will not + change for the duration of the program's execution. No code in the Python + interpreter will change the contents of this storage. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + .. deprecated-removed:: 3.11 3.15 + + +.. c:function:: const char* Py_GetVersion() + + Return the version of this Python interpreter. This is a string that looks + something like :: + + "3.0a5+ (py3k:63103M, May 12 2008, 00:53:55) \n[GCC 4.2.3]" + + .. index:: single: version (in module sys) + + The first word (up to the first space character) is the current Python version; + the first characters are the major and minor version separated by a + period. The returned string points into static storage; the caller should not + modify its value. The value is available to Python code as :data:`sys.version`. + + See also the :c:var:`Py_Version` constant. + + +.. c:function:: const char* Py_GetPlatform() + + .. index:: single: platform (in module sys) + + Return the platform identifier for the current platform. On Unix, this is + formed from the "official" name of the operating system, converted to lower + case, followed by the major revision number; e.g., for Solaris 2.x, which is + also known as SunOS 5.x, the value is ``'sunos5'``. On macOS, it is + ``'darwin'``. On Windows, it is ``'win'``. The returned string points into + static storage; the caller should not modify its value. The value is available + to Python code as ``sys.platform``. + + +.. c:function:: const char* Py_GetCopyright() + + Return the official copyright string for the current Python version, for example + + ``'Copyright 1991-1995 Stichting Mathematisch Centrum, Amsterdam'`` + + .. index:: single: copyright (in module sys) + + The returned string points into static storage; the caller should not modify its + value. The value is available to Python code as ``sys.copyright``. + + +.. c:function:: const char* Py_GetCompiler() + + Return an indication of the compiler used to build the current Python version, + in square brackets, for example:: + + "[GCC 2.7.2.2]" + + .. index:: single: version (in module sys) + + The returned string points into static storage; the caller should not modify its + value. The value is available to Python code as part of the variable + ``sys.version``. + + +.. c:function:: const char* Py_GetBuildInfo() + + Return information about the sequence number and build date and time of the + current Python interpreter instance, for example :: + + "#67, Aug 1 1997, 22:34:28" + + .. index:: single: version (in module sys) + + The returned string points into static storage; the caller should not modify its + value. The value is available to Python code as part of the variable + ``sys.version``. + + +.. c:function:: void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) + + .. index:: + single: main() + single: Py_FatalError() + single: argv (in module sys) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.argv`, :c:member:`PyConfig.parse_argv` and + :c:member:`PyConfig.safe_path` should be used instead, see :ref:`Python + Initialization Configuration `. + + Set :data:`sys.argv` based on *argc* and *argv*. These parameters are + similar to those passed to the program's :c:func:`main` function with the + difference that the first entry should refer to the script file to be + executed rather than the executable hosting the Python interpreter. If there + isn't a script that will be run, the first entry in *argv* can be an empty + string. If this function fails to initialize :data:`sys.argv`, a fatal + condition is signalled using :c:func:`Py_FatalError`. + + If *updatepath* is zero, this is all the function does. If *updatepath* + is non-zero, the function also modifies :data:`sys.path` according to the + following algorithm: + + - If the name of an existing script is passed in ``argv[0]``, the absolute + path of the directory where the script is located is prepended to + :data:`sys.path`. + - Otherwise (that is, if *argc* is ``0`` or ``argv[0]`` doesn't point + to an existing file name), an empty string is prepended to + :data:`sys.path`, which is the same as prepending the current working + directory (``"."``). + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` + members of the :ref:`Python Initialization Configuration `. + + .. note:: + It is recommended that applications embedding the Python interpreter + for purposes other than executing a single script pass ``0`` as *updatepath*, + and update :data:`sys.path` themselves if desired. + See :cve:`2008-5983`. + + On versions before 3.1.3, you can achieve the same effect by manually + popping the first :data:`sys.path` element after having called + :c:func:`PySys_SetArgv`, for example using:: + + PyRun_SimpleString("import sys; sys.path.pop(0)\n"); + + .. versionadded:: 3.1.3 + + .. deprecated-removed:: 3.11 3.15 + + +.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.argv` and :c:member:`PyConfig.parse_argv` should be used + instead, see :ref:`Python Initialization Configuration `. + + This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set + to ``1`` unless the :program:`python` interpreter was started with the + :option:`-I`. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` + members of the :ref:`Python Initialization Configuration `. + + .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. + + .. deprecated-removed:: 3.11 3.15 + + +.. c:function:: void Py_SetPythonHome(const wchar_t *home) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.home` should be used instead, see :ref:`Python + Initialization Configuration `. + + Set the default "home" directory, that is, the location of the standard + Python libraries. See :envvar:`PYTHONHOME` for the meaning of the + argument string. + + The argument should point to a zero-terminated character string in static + storage whose contents will not change for the duration of the program's + execution. No code in the Python interpreter will change the contents of + this storage. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + .. deprecated-removed:: 3.11 3.15 diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index acce3dc215d..500f2818e2e 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -107,6 +107,46 @@ header files properly declare the entry points to be ``extern "C"``. As a result there is no need to do anything special to use the API from C++. +.. _capi-system-includes: + +System includes +--------------- + + :file:`Python.h` includes several standard header files. + C extensions should include the standard headers that they use, + and should not rely on these implicit includes. + The implicit includes are: + + * ```` + * ```` (on Windows) + * ```` + * ```` + * ```` + * ```` + * ```` + * ```` + * ```` (if present) + + The following are included for backwards compatibility, unless using + :ref:`Limited API ` 3.13 or newer: + + * ```` + * ```` (on POSIX) + + The following are included for backwards compatibility, unless using + :ref:`Limited API ` 3.11 or newer: + + * ```` + * ```` + * ```` + +.. note:: + + Since Python may define some pre-processor definitions which affect the standard + headers on some systems, you *must* include :file:`Python.h` before any standard + headers are included. + + Useful macros ============= @@ -116,18 +156,279 @@ defined closer to where they are useful (for example, :c:macro:`Py_RETURN_NONE`, Others of a more general utility are defined here. This is not necessarily a complete listing. +.. c:macro:: Py_CAN_START_THREADS + + If this macro is defined, then the current system is able to start threads. + + Currently, all systems supported by CPython (per :pep:`11`), with the + exception of some WebAssembly platforms, support starting threads. + + .. versionadded:: 3.13 + +.. c:macro:: Py_GETENV(s) + + Like :samp:`getenv({s})`, but returns ``NULL`` if :option:`-E` was passed + on the command line (see :c:member:`PyConfig.use_environment`). + + +Docstring macros +---------------- + +.. c:macro:: PyDoc_STRVAR(name, str) + + Creates a variable with name *name* that can be used in docstrings. + If Python is built without docstrings (:option:`--without-doc-strings`), + the value will be an empty string. + + Example:: + + PyDoc_STRVAR(pop_doc, "Remove and return the rightmost element."); + + static PyMethodDef deque_methods[] = { + // ... + {"pop", (PyCFunction)deque_pop, METH_NOARGS, pop_doc}, + // ... + } + + Expands to :samp:`PyDoc_VAR({name}) = PyDoc_STR({str})`. + +.. c:macro:: PyDoc_STR(str) + + Expands to the given input string, or an empty string + if docstrings are disabled (:option:`--without-doc-strings`). + + Example:: + + static PyMethodDef pysqlite_row_methods[] = { + {"keys", (PyCFunction)pysqlite_row_keys, METH_NOARGS, + PyDoc_STR("Returns the keys of the row.")}, + {NULL, NULL} + }; + +.. c:macro:: PyDoc_VAR(name) + + Declares a static character array variable with the given *name*. + Expands to :samp:`static const char {name}[]` + + For example:: + + PyDoc_VAR(python_doc) = PyDoc_STR( + "A genus of constricting snakes in the Pythonidae family native " + "to the tropics and subtropics of the Eastern Hemisphere."); + + +General utility macros +---------------------- + +The following macros are for common tasks not specific to Python. + +.. c:macro:: Py_UNUSED(arg) + + Use this for unused arguments in a function definition to silence compiler + warnings. Example: ``int func(int a, int Py_UNUSED(b)) { return a; }``. + + .. versionadded:: 3.4 + +.. c:macro:: Py_GCC_ATTRIBUTE(name) + + Use a GCC attribute *name*, hiding it from compilers that don't support GCC + attributes (such as MSVC). + + This expands to :samp:`__attribute__(({name)})` on a GCC compiler, + and expands to nothing on compilers that don't support GCC attributes. + + +Numeric utilities +^^^^^^^^^^^^^^^^^ .. c:macro:: Py_ABS(x) Return the absolute value of ``x``. + The argument may be evaluated more than once. + Consequently, do not pass an expression with side-effects directly + to this macro. + + If the result cannot be represented (for example, if ``x`` has + :c:macro:`!INT_MIN` value for :c:expr:`int` type), the behavior is + undefined. + + Corresponds roughly to :samp:`(({x}) < 0 ? -({x}) : ({x}))` + .. versionadded:: 3.3 +.. c:macro:: Py_MAX(x, y) + Py_MIN(x, y) + + Return the larger or smaller of the arguments, respectively. + + Any arguments may be evaluated more than once. + Consequently, do not pass an expression with side-effects directly + to this macro. + + :c:macro:`!Py_MAX` corresponds roughly to + :samp:`((({x}) > ({y})) ? ({x}) : ({y}))`. + + .. versionadded:: 3.3 + +.. c:macro:: Py_ARITHMETIC_RIGHT_SHIFT(type, integer, positions) + + Similar to :samp:`{integer} >> {positions}`, but forces sign extension, + as the C standard does not define whether a right-shift of a signed + integer will perform sign extension or a zero-fill. + + *integer* should be any signed integer type. + *positions* is the number of positions to shift to the right. + + Both *integer* and *positions* can be evaluated more than once; + consequently, avoid directly passing a function call or some other + operation with side-effects to this macro. Instead, store the result as a + variable and then pass it. + + *type* is unused and only kept for backwards compatibility. Historically, + *type* was used to cast *integer*. + + .. versionchanged:: 3.1 + + This macro is now valid for all signed integer types, not just those for + which ``unsigned type`` is legal. As a result, *type* is no longer + used. + +.. c:macro:: Py_CHARMASK(c) + + Argument must be a character or an integer in the range [-128, 127] or [0, + 255]. This macro returns ``c`` cast to an ``unsigned char``. + + +Assertion utilities +^^^^^^^^^^^^^^^^^^^ + +.. c:macro:: Py_UNREACHABLE() + + Use this when you have a code path that cannot be reached by design. + For example, in the ``default:`` clause in a ``switch`` statement for which + all possible values are covered in ``case`` statements. Use this in places + where you might be tempted to put an ``assert(0)`` or ``abort()`` call. + + In release mode, the macro helps the compiler to optimize the code, and + avoids a warning about unreachable code. For example, the macro is + implemented with ``__builtin_unreachable()`` on GCC in release mode. + + In debug mode, and on unsupported compilers, the macro expands to a call to + :c:func:`Py_FatalError`. + + A use for ``Py_UNREACHABLE()`` is following a call to a function that + never returns but that is not declared ``_Noreturn``. + + If a code path is very unlikely code but can be reached under exceptional + case, this macro must not be used. For example, under low memory condition + or if a system call returns a value out of the expected range. In this + case, it's better to report the error to the caller. If the error cannot + be reported to caller, :c:func:`Py_FatalError` can be used. + + .. versionadded:: 3.7 + +.. c:macro:: Py_SAFE_DOWNCAST(value, larger, smaller) + + Cast *value* to type *smaller* from type *larger*, validating that no + information was lost. + + On release builds of Python, this is roughly equivalent to + :samp:`(({smaller}) {value})` + (in C++, :samp:`static_cast<{smaller}>({value})` will be used instead). + + On debug builds (implying that :c:macro:`Py_DEBUG` is defined), this asserts + that no information was lost with the cast from *larger* to *smaller*. + + *value*, *larger*, and *smaller* may all be evaluated more than once in the + expression; consequently, do not pass an expression with side-effects + directly to this macro. + +.. c:macro:: Py_BUILD_ASSERT(cond) + + Asserts a compile-time condition *cond*, as a statement. + The build will fail if the condition is false or cannot be evaluated at compile time. + + Corresponds roughly to :samp:`static_assert({cond})` on C23 and above. + + For example:: + + Py_BUILD_ASSERT(sizeof(PyTime_t) == sizeof(int64_t)); + + .. versionadded:: 3.3 + +.. c:macro:: Py_BUILD_ASSERT_EXPR(cond) + + Asserts a compile-time condition *cond*, as an expression that evaluates to ``0``. + The build will fail if the condition is false or cannot be evaluated at compile time. + + For example:: + + #define foo_to_char(foo) \ + ((char *)(foo) + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) + + .. versionadded:: 3.3 + + +Type size utilities +^^^^^^^^^^^^^^^^^^^ + +.. c:macro:: Py_ARRAY_LENGTH(array) + + Compute the length of a statically allocated C array at compile time. + + The *array* argument must be a C array with a size known at compile time. + Passing an array with an unknown size, such as a heap-allocated array, + will result in a compilation error on some compilers, or otherwise produce + incorrect results. + + This is roughly equivalent to:: + + sizeof(array) / sizeof((array)[0]) + +.. c:macro:: Py_MEMBER_SIZE(type, member) + + Return the size of a structure (*type*) *member* in bytes. + + Corresponds roughly to :samp:`sizeof((({type} *)NULL)->{member})`. + + .. versionadded:: 3.6 + + +Macro definition utilities +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:macro:: Py_FORCE_EXPANSION(X) + + This is equivalent to :samp:`{X}`, which is useful for token-pasting in + macros, as macro expansions in *X* are forcefully evaluated by the + preprocessor. + +.. c:macro:: Py_STRINGIFY(x) + + Convert ``x`` to a C string. For example, ``Py_STRINGIFY(123)`` returns + ``"123"``. + + .. versionadded:: 3.4 + + +Declaration utilities +--------------------- + +The following macros can be used in declarations. +They are most useful for defining the C API itself, and have limited use +for extension authors. +Most of them expand to compiler-specific spellings of common extensions +to the C language. + .. c:macro:: Py_ALWAYS_INLINE Ask the compiler to always inline a static inline function. The compiler can ignore it and decide to not inline the function. + Corresponds to ``always_inline`` attribute in GCC and ``__forceinline`` + in MSVC. + It can be used to inline performance critical static inline functions when building Python in debug mode with function inlining disabled. For example, MSC disables function inlining when building in debug mode. @@ -145,15 +446,24 @@ complete listing. .. versionadded:: 3.11 -.. c:macro:: Py_CHARMASK(c) +.. c:macro:: Py_NO_INLINE - Argument must be a character or an integer in the range [-128, 127] or [0, - 255]. This macro returns ``c`` cast to an ``unsigned char``. + Disable inlining on a function. For example, it reduces the C stack + consumption: useful on LTO+PGO builds which heavily inline code (see + :issue:`33720`). + + Corresponds to the ``noinline`` attribute/specification on GCC and MSVC. + + Usage:: + + Py_NO_INLINE static int random(void) { return 4; } + + .. versionadded:: 3.11 .. c:macro:: Py_DEPRECATED(version) - Use this for deprecated declarations. The macro must be placed before the - symbol name. + Use this to declare APIs that were deprecated in a specific CPython version. + The macro must be placed before the symbol name. Example:: @@ -162,110 +472,153 @@ complete listing. .. versionchanged:: 3.8 MSVC support was added. -.. c:macro:: Py_GETENV(s) +.. c:macro:: Py_LOCAL(type) - Like ``getenv(s)``, but returns ``NULL`` if :option:`-E` was passed on the - command line (see :c:member:`PyConfig.use_environment`). + Declare a function returning the specified *type* using a fast-calling + qualifier for functions that are local to the current file. + Semantically, this is equivalent to :samp:`static {type}`. -.. c:macro:: Py_MAX(x, y) +.. c:macro:: Py_LOCAL_INLINE(type) - Return the maximum value between ``x`` and ``y``. + Equivalent to :c:macro:`Py_LOCAL` but additionally requests the function + be inlined. - .. versionadded:: 3.3 +.. c:macro:: Py_LOCAL_SYMBOL -.. c:macro:: Py_MEMBER_SIZE(type, member) + Macro used to declare a symbol as local to the shared library (hidden). + On supported platforms, it ensures the symbol is not exported. - Return the size of a structure (``type``) ``member`` in bytes. + On compatible versions of GCC/Clang, it + expands to ``__attribute__((visibility("hidden")))``. - .. versionadded:: 3.6 +.. c:macro:: Py_EXPORTED_SYMBOL -.. c:macro:: Py_MIN(x, y) + Macro used to declare a symbol (function or data) as exported. + On Windows, this expands to ``__declspec(dllexport)``. + On compatible versions of GCC/Clang, it + expands to ``__attribute__((visibility("default")))``. + This macro is for defining the C API itself; extension modules should not use it. - Return the minimum value between ``x`` and ``y``. - .. versionadded:: 3.3 +.. c:macro:: Py_IMPORTED_SYMBOL -.. c:macro:: Py_NO_INLINE + Macro used to declare a symbol as imported. + On Windows, this expands to ``__declspec(dllimport)``. + This macro is for defining the C API itself; extension modules should not use it. - Disable inlining on a function. For example, it reduces the C stack - consumption: useful on LTO+PGO builds which heavily inline code (see - :issue:`33720`). - Usage:: +.. c:macro:: PyAPI_FUNC(type) - Py_NO_INLINE static int random(void) { return 4; } + Macro used by CPython to declare a function as part of the C API. + Its expansion depends on the platform and build configuration. + This macro is intended for defining CPython's C API itself; + extension modules should not use it for their own symbols. - .. versionadded:: 3.11 -.. c:macro:: Py_STRINGIFY(x) +.. c:macro:: PyAPI_DATA(type) - Convert ``x`` to a C string. E.g. ``Py_STRINGIFY(123)`` returns - ``"123"``. + Macro used by CPython to declare a public global variable as part of the C API. + Its expansion depends on the platform and build configuration. + This macro is intended for defining CPython's C API itself; + extension modules should not use it for their own symbols. - .. versionadded:: 3.4 -.. c:macro:: Py_UNREACHABLE() +Outdated macros +--------------- - Use this when you have a code path that cannot be reached by design. - For example, in the ``default:`` clause in a ``switch`` statement for which - all possible values are covered in ``case`` statements. Use this in places - where you might be tempted to put an ``assert(0)`` or ``abort()`` call. +The following :term:`soft deprecated` macros have been used to features that +have been standardized in C11 (or previous standards). - In release mode, the macro helps the compiler to optimize the code, and - avoids a warning about unreachable code. For example, the macro is - implemented with ``__builtin_unreachable()`` on GCC in release mode. +.. c:macro:: Py_ALIGNED(num) - A use for ``Py_UNREACHABLE()`` is following a call a function that - never returns but that is not declared :c:macro:`_Py_NO_RETURN`. + On some GCC-like compilers, specify alignment to *num* bytes. + This does nothing on other compilers. - If a code path is very unlikely code but can be reached under exceptional - case, this macro must not be used. For example, under low memory condition - or if a system call returns a value out of the expected range. In this - case, it's better to report the error to the caller. If the error cannot - be reported to caller, :c:func:`Py_FatalError` can be used. + Use the standard ``alignas`` specifier rather than this macro. - .. versionadded:: 3.7 + .. soft-deprecated:: 3.15 -.. c:macro:: Py_UNUSED(arg) +.. c:macro:: PY_FORMAT_SIZE_T - Use this for unused arguments in a function definition to silence compiler - warnings. Example: ``int func(int a, int Py_UNUSED(b)) { return a; }``. + The :c:func:`printf` formatting modifier for :c:type:`size_t`. + Use ``"z"`` directly instead. - .. versionadded:: 3.4 + .. soft-deprecated:: 3.15 -.. c:macro:: PyDoc_STRVAR(name, str) +.. c:macro:: Py_LL(number) + Py_ULL(number) - Creates a variable with name ``name`` that can be used in docstrings. - If Python is built without docstrings, the value will be empty. + Use *number* as a ``long long`` or ``unsigned long long`` integer literal, + respectively. - Use :c:macro:`PyDoc_STRVAR` for docstrings to support building - Python without docstrings, as specified in :pep:`7`. + Expands to *number* followed by ``LL`` or ``LLU``, respectively, but will + expand to some compiler-specific suffixes on some older compilers. - Example:: + Consider using the C99 standard suffixes ``LL`` and ``LLU`` directly. - PyDoc_STRVAR(pop_doc, "Remove and return the rightmost element."); + .. soft-deprecated:: 3.15 - static PyMethodDef deque_methods[] = { - // ... - {"pop", (PyCFunction)deque_pop, METH_NOARGS, pop_doc}, - // ... - } +.. c:macro:: PY_LONG_LONG + PY_INT32_T + PY_UINT32_T + PY_INT64_T + PY_UINT64_T -.. c:macro:: PyDoc_STR(str) + Aliases for the types :c:type:`!long long`, :c:type:`!int32_t`, + :c:type:`!uint32_t`. :c:type:`!int64_t` and :c:type:`!uint64_t`, + respectively. + Historically, these types needed compiler-specific extensions. - Creates a docstring for the given input string or an empty string - if docstrings are disabled. + .. soft-deprecated:: 3.15 - Use :c:macro:`PyDoc_STR` in specifying docstrings to support - building Python without docstrings, as specified in :pep:`7`. +.. c:macro:: PY_LLONG_MIN + PY_LLONG_MAX + PY_ULLONG_MAX + PY_SIZE_MAX - Example:: + Aliases for the values :c:macro:`!LLONG_MIN`, :c:macro:`!LLONG_MAX`, + :c:macro:`!ULLONG_MAX`, and :c:macro:`!SIZE_MAX`, respectively. + Use these standard names instead. - static PyMethodDef pysqlite_row_methods[] = { - {"keys", (PyCFunction)pysqlite_row_keys, METH_NOARGS, - PyDoc_STR("Returns the keys of the row.")}, - {NULL, NULL} - }; + The required header, ````, + :ref:`is included ` in ``Python.h``. + + .. soft-deprecated:: 3.15 + +.. c:macro:: Py_MEMCPY(dest, src, n) + + This is an alias to :c:func:`!memcpy`. + + .. soft-deprecated:: 3.14 + Use :c:func:`!memcpy` directly instead. + +.. c:macro:: Py_UNICODE_SIZE + + Size of the :c:type:`!wchar_t` type. + Use ``sizeof(wchar_t)`` or ``WCHAR_WIDTH/8`` instead. + + The required header for the latter, ````, + :ref:`is included ` in ``Python.h``. + + .. soft-deprecated:: 3.15 + +.. c:macro:: Py_UNICODE_WIDE + + Defined if ``wchar_t`` can hold a Unicode character (UCS-4). + Use ``sizeof(wchar_t) >= 4`` instead + + .. soft-deprecated:: 3.15 + +.. c:macro:: Py_VA_COPY + + This is an alias to the C99-standard ``va_copy`` function. + + Historically, this would use a compiler-specific method to copy a ``va_list``. + + .. versionchanged:: 3.6 + This is now an alias to ``va_copy``. + + .. soft-deprecated:: 3.15 .. _api-objects: diff --git a/Doc/c-api/iter.rst b/Doc/c-api/iter.rst index bf9df62c6f1..6cfd24c5ae6 100644 --- a/Doc/c-api/iter.rst +++ b/Doc/c-api/iter.rst @@ -54,6 +54,6 @@ There are two functions specifically for working with iterators. - ``PYGEN_RETURN`` if iterator returns. Return value is returned via *presult*. - ``PYGEN_NEXT`` if iterator yields. Yielded value is returned via *presult*. - - ``PYGEN_ERROR`` if iterator has raised and exception. *presult* is set to ``NULL``. + - ``PYGEN_ERROR`` if iterator has raised an exception. *presult* is set to ``NULL``. .. versionadded:: 3.10 diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst index 6b7ba8c9979..bfbfe3c9279 100644 --- a/Doc/c-api/iterator.rst +++ b/Doc/c-api/iterator.rst @@ -50,3 +50,72 @@ sentinel value is returned. callable object that can be called with no parameters; each call to it should return the next item in the iteration. When *callable* returns a value equal to *sentinel*, the iteration will be terminated. + + +Range Objects +^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PyRange_Type + + The type object for :class:`range` objects. + + +.. c:function:: int PyRange_Check(PyObject *o) + + Return true if the object *o* is an instance of a :class:`range` object. + This function always succeeds. + + +Builtin Iterator Types +^^^^^^^^^^^^^^^^^^^^^^ + +These are built-in iteration types that are included in Python's C API, but +provide no additional functions. They are here for completeness. + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * C type + * Python type + * * .. c:var:: PyTypeObject PyEnum_Type + * :py:class:`enumerate` + * * .. c:var:: PyTypeObject PyFilter_Type + * :py:class:`filter` + * * .. c:var:: PyTypeObject PyMap_Type + * :py:class:`map` + * * .. c:var:: PyTypeObject PyReversed_Type + * :py:class:`reversed` + * * .. c:var:: PyTypeObject PyZip_Type + * :py:class:`zip` + + +Other Iterator Objects +^^^^^^^^^^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PyByteArrayIter_Type +.. c:var:: PyTypeObject PyBytesIter_Type +.. c:var:: PyTypeObject PyListIter_Type +.. c:var:: PyTypeObject PyListRevIter_Type +.. c:var:: PyTypeObject PySetIter_Type +.. c:var:: PyTypeObject PyTupleIter_Type +.. c:var:: PyTypeObject PyRangeIter_Type +.. c:var:: PyTypeObject PyLongRangeIter_Type +.. c:var:: PyTypeObject PyDictIterKey_Type +.. c:var:: PyTypeObject PyDictRevIterKey_Type +.. c:var:: PyTypeObject PyDictIterValue_Type +.. c:var:: PyTypeObject PyDictRevIterValue_Type +.. c:var:: PyTypeObject PyDictIterItem_Type +.. c:var:: PyTypeObject PyDictRevIterItem_Type +.. c:var:: PyTypeObject PyODictIter_Type + + Type objects for iterators of various built-in objects. + + Do not create instances of these directly; prefer calling + :c:func:`PyObject_GetIter` instead. + + Note that there is no guarantee that a given built-in type uses a given iterator + type. For example, iterating over :class:`range` will use one of two iterator + types depending on the size of the range. Other types may start using a + similar scheme in the future, without warning. diff --git a/Doc/c-api/lifecycle.rst b/Doc/c-api/lifecycle.rst index 5a170862a26..531c4080a01 100644 --- a/Doc/c-api/lifecycle.rst +++ b/Doc/c-api/lifecycle.rst @@ -256,6 +256,8 @@ To allocate and free memory, see :ref:`allocating-objects`. collection (i.e., the :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set); this may change in the future. + .. versionadded:: 3.4 + .. c:function:: int PyObject_CallFinalizerFromDealloc(PyObject *op) @@ -266,6 +268,8 @@ To allocate and free memory, see :ref:`allocating-objects`. should happen. Otherwise, this function returns 0 and destruction can continue normally. + .. versionadded:: 3.4 + .. seealso:: :c:member:`~PyTypeObject.tp_dealloc` for example code. diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 758415a76e5..8f560699d35 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -74,11 +74,25 @@ List Objects Like :c:func:`PyList_GetItemRef`, but returns a :term:`borrowed reference` instead of a :term:`strong reference`. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the list concurrently. Prefer :c:func:`PyList_GetItemRef`, which returns + a :term:`strong reference`. + .. c:function:: PyObject* PyList_GET_ITEM(PyObject *list, Py_ssize_t i) Similar to :c:func:`PyList_GetItem`, but without error checking. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the list concurrently. Prefer :c:func:`PyList_GetItemRef`, which returns + a :term:`strong reference`. + .. c:function:: int PyList_SetItem(PyObject *list, Py_ssize_t index, PyObject *item) @@ -108,6 +122,14 @@ List Objects is being replaced; any reference in *list* at position *i* will be leaked. + .. note:: + + In the :term:`free-threaded build`, this macro has no internal + synchronization. It is normally only used to fill in new lists where no + other thread has a reference to the list. If the list may be shared, + use :c:func:`PyList_SetItem` instead, which uses a :term:`per-object + lock`. + .. c:function:: int PyList_Insert(PyObject *list, Py_ssize_t index, PyObject *item) @@ -138,6 +160,12 @@ List Objects Return ``0`` on success, ``-1`` on failure. Indexing from the end of the list is not supported. + .. note:: + + In the :term:`free-threaded build`, when *itemlist* is a :class:`list`, + both *list* and *itemlist* are locked for the duration of the operation. + For other iterables (or ``NULL``), only *list* is locked. + .. c:function:: int PyList_Extend(PyObject *list, PyObject *iterable) @@ -150,6 +178,14 @@ List Objects .. versionadded:: 3.13 + .. note:: + + In the :term:`free-threaded build`, when *iterable* is a :class:`list`, + :class:`set`, :class:`dict`, or dict view, both *list* and *iterable* + (or its underlying dict) are locked for the duration of the operation. + For other iterables, only *list* is locked; *iterable* may be + concurrently modified by another thread. + .. c:function:: int PyList_Clear(PyObject *list) @@ -168,6 +204,14 @@ List Objects Sort the items of *list* in place. Return ``0`` on success, ``-1`` on failure. This is equivalent to ``list.sort()``. + .. note:: + + In the :term:`free-threaded build`, element comparison via + :meth:`~object.__lt__` can execute arbitrary Python code, during which + the :term:`per-object lock` may be temporarily released. For built-in + types (:class:`str`, :class:`int`, :class:`float`), the lock is not + released during comparison. + .. c:function:: int PyList_Reverse(PyObject *list) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 8370dcecad3..60e3ae4a064 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -43,7 +43,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. impl-detail:: CPython keeps an array of integer objects for all integers - between ``-5`` and ``256``. When you create an int in that range + between ``-5`` and ``1024``. When you create an int in that range you actually just get back a reference to the existing object. @@ -161,6 +161,17 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.13 +.. c:macro:: PyLong_FromPid(pid) + + Macro for creating a Python integer from a process identifier. + + This can be defined as an alias to :c:func:`PyLong_FromLong` or + :c:func:`PyLong_FromLongLong`, depending on the size of the system's + PID type. + + .. versionadded:: 3.2 + + .. c:function:: long PyLong_AsLong(PyObject *obj) .. index:: @@ -186,12 +197,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: long PyLong_AS_LONG(PyObject *obj) - A :term:`soft deprecated` alias. Exactly equivalent to the preferred ``PyLong_AsLong``. In particular, it can fail with :exc:`OverflowError` or another exception. - .. deprecated:: 3.14 - The function is soft deprecated. + .. soft-deprecated:: 3.14 .. c:function:: int PyLong_AsInt(PyObject *obj) @@ -442,8 +451,8 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Otherwise, returns the number of bytes required to store the value. If this is equal to or less than *n_bytes*, the entire value was copied. - All *n_bytes* of the buffer are written: large buffers are padded with - zeroes. + All *n_bytes* of the buffer are written: remaining bytes filled by + copies of the sign bit. If the returned value is greater than *n_bytes*, the value was truncated: as many of the lowest bits of the value as could fit are written, @@ -575,6 +584,17 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.13 +.. c:macro:: PyLong_AsPid(pid) + + Macro for converting a Python integer into a process identifier. + + This can be defined as an alias to :c:func:`PyLong_AsLong`, + :c:func:`PyLong_FromLongLong`, or :c:func:`PyLong_AsInt`, depending on the + size of the system's PID type. + + .. versionadded:: 3.2 + + .. c:function:: int PyLong_GetSign(PyObject *obj, int *sign) Get the sign of the integer object *obj*. @@ -665,7 +685,7 @@ Export API .. versionadded:: 3.14 -.. c:struct:: PyLongLayout +.. c:type:: PyLongLayout Layout of an array of "digits" ("limbs" in the GMP terminology), used to represent absolute value for arbitrary precision integers. @@ -705,7 +725,7 @@ Export API Get the native layout of Python :class:`int` objects. - See the :c:struct:`PyLongLayout` structure. + See the :c:type:`PyLongLayout` structure. The function must not be called before Python initialization nor after Python finalization. The returned layout is valid until Python is @@ -713,7 +733,7 @@ Export API in a process, and so it can be cached. -.. c:struct:: PyLongExport +.. c:type:: PyLongExport Export of a Python :class:`int` object. @@ -747,7 +767,7 @@ Export API Export a Python :class:`int` object. - *export_long* must point to a :c:struct:`PyLongExport` structure allocated + *export_long* must point to a :c:type:`PyLongExport` structure allocated by the caller. It must not be ``NULL``. On success, fill in *\*export_long* and return ``0``. @@ -777,7 +797,7 @@ The :c:type:`PyLongWriter` API can be used to import an integer. .. versionadded:: 3.14 -.. c:struct:: PyLongWriter +.. c:type:: PyLongWriter A Python :class:`int` writer instance. @@ -805,7 +825,7 @@ The :c:type:`PyLongWriter` API can be used to import an integer. The layout of *digits* is described by :c:func:`PyLong_GetNativeLayout`. Digits must be in the range [``0``; ``(1 << bits_per_digit) - 1``] - (where the :c:struct:`~PyLongLayout.bits_per_digit` is the number of bits + (where the :c:type:`~PyLongLayout.bits_per_digit` is the number of bits per digit). Any unused most significant digits must be set to ``0``. @@ -833,3 +853,31 @@ The :c:type:`PyLongWriter` API can be used to import an integer. If *writer* is ``NULL``, no operation is performed. The writer instance and the *digits* array are invalid after the call. + + +Deprecated API +^^^^^^^^^^^^^^ + +These macros are :term:`soft deprecated`. They describe parameters +of the internal representation of :c:type:`PyLongObject` instances. + +Use :c:func:`PyLong_GetNativeLayout` instead, along with :c:func:`PyLong_Export` +to read integer data or :c:type:`PyLongWriter` to write it. +These currently use the same layout, but are designed to continue working correctly +even if CPython's internal integer representation changes. + + +.. c:macro:: PyLong_SHIFT + + This is equivalent to :c:member:`~PyLongLayout.bits_per_digit` in + the output of :c:func:`PyLong_GetNativeLayout`. + + +.. c:macro:: PyLong_BASE + + This is currently equivalent to :c:expr:`1 << PyLong_SHIFT`. + + +.. c:macro:: PyLong_MASK + + This is currently equivalent to :c:expr:`(1 << PyLong_SHIFT) - 1` diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst index 1f55c0aa955..2476ebb9b69 100644 --- a/Doc/c-api/mapping.rst +++ b/Doc/c-api/mapping.rst @@ -102,7 +102,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. note:: - Exceptions which occur when this calls :meth:`~object.__getitem__` + Exceptions which occur when this calls the :meth:`~object.__getitem__` method are silently ignored. For proper error handling, use :c:func:`PyMapping_HasKeyWithError`, :c:func:`PyMapping_GetOptionalItem` or :c:func:`PyObject_GetItem()` instead. @@ -116,7 +116,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. note:: - Exceptions that occur when this calls :meth:`~object.__getitem__` + Exceptions that occur when this calls the :meth:`~object.__getitem__` method or while creating the temporary :class:`str` object are silently ignored. For proper error handling, use :c:func:`PyMapping_HasKeyStringWithError`, diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index 61218a1bf6f..668a163b2df 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -82,7 +82,7 @@ The following functions allow marshalled values to be read back in. assumes that no further objects will be read from the file, allowing it to aggressively load file data into memory so that the de-serialization can operate from data in memory rather than reading a byte at a time from the - file. Only use these variant if you are certain that you won't be reading + file. Only use this variant if you are certain that you won't be reading anything else from the file. On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index df1bb0ce370..9f84e4bc6df 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -7,10 +7,6 @@ Memory Management ***************** -.. sectionauthor:: Vladimir Marangozov - - - .. _memoryoverview: Overview @@ -102,7 +98,7 @@ All allocating functions belong to one of three different "domains" (see also strategies and are optimized for different purposes. The specific details on how every domain allocates memory or what internal functions each domain calls is considered an implementation detail, but for debugging purposes a simplified -table can be found at :ref:`here `. +table can be found at :ref:`default-memory-allocators`. The APIs used to allocate and free a block of memory must be from the same domain. For example, :c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. @@ -208,8 +204,11 @@ The following function sets, modeled after the ANSI C standard, but specifying behavior when requesting zero bytes, are available for allocating and releasing memory from the Python heap. -The :ref:`default memory allocator ` uses the -:ref:`pymalloc memory allocator `. +In the GIL-enabled build (default build) the +:ref:`default memory allocator ` uses the +:ref:`pymalloc memory allocator `, whereas in the +:term:`free-threaded build`, the default is the +:ref:`mimalloc memory allocator ` instead. .. warning:: @@ -219,6 +218,11 @@ The :ref:`default memory allocator ` uses the The default allocator is now pymalloc instead of system :c:func:`malloc`. +.. versionchanged:: 3.13 + + In the :term:`free-threaded ` build, the default allocator + is now :ref:`mimalloc `. + .. c:function:: void* PyMem_Malloc(size_t n) Allocates *n* bytes and returns a pointer of type :c:expr:`void*` to the @@ -293,17 +297,39 @@ The following type-oriented macros are provided for convenience. Note that Same as :c:func:`PyMem_Free`. -In addition, the following macro sets are provided for calling the Python memory -allocator directly, without involving the C API functions listed above. However, -note that their use does not preserve binary compatibility across Python -versions and is therefore deprecated in extension modules. -* ``PyMem_MALLOC(size)`` -* ``PyMem_NEW(type, size)`` -* ``PyMem_REALLOC(ptr, size)`` -* ``PyMem_RESIZE(ptr, type, size)`` -* ``PyMem_FREE(ptr)`` -* ``PyMem_DEL(ptr)`` +Deprecated aliases +------------------ + +These are :term:`soft deprecated` aliases to existing functions and macros. +They exist solely for backwards compatibility. + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * Deprecated alias + * Corresponding function or macro + * * .. c:macro:: PyMem_MALLOC(size) + * :c:func:`PyMem_Malloc` + * * .. c:macro:: PyMem_NEW(type, size) + * :c:macro:`PyMem_New` + * * .. c:macro:: PyMem_REALLOC(ptr, size) + * :c:func:`PyMem_Realloc` + * * .. c:macro:: PyMem_RESIZE(ptr, type, size) + * :c:macro:`PyMem_Resize` + * * .. c:macro:: PyMem_FREE(ptr) + * :c:func:`PyMem_Free` + * * .. c:macro:: PyMem_DEL(ptr) + * :c:func:`PyMem_Free` + +.. versionchanged:: 3.4 + + The macros are now aliases of the corresponding functions and macros. + Previously, their behavior was the same, but their use did not necessarily + preserve binary compatibility across Python versions. + +.. deprecated:: 2.0 .. _objectinterface: @@ -322,7 +348,9 @@ memory from the Python heap. the :ref:`Customize Memory Allocators ` section. The :ref:`default object allocator ` uses the -:ref:`pymalloc memory allocator `. +:ref:`pymalloc memory allocator `. In the +:term:`free-threaded ` build, the default is the +:ref:`mimalloc memory allocator ` instead. .. warning:: @@ -402,14 +430,16 @@ Default Memory Allocators Default memory allocators: -=============================== ==================== ================== ===================== ==================== -Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc -=============================== ==================== ================== ===================== ==================== -Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc`` -Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug -Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` -Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug -=============================== ==================== ================== ===================== ==================== +=================================== ======================= ==================== ====================== ====================== +Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc +=================================== ======================= ==================== ====================== ====================== +Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc`` +Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug +Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` +Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug +Free-threaded build ``"mimalloc"`` ``mimalloc`` ``mimalloc`` ``mimalloc`` +Free-threaded debug build ``"mimalloc_debug"`` ``mimalloc`` + debug ``mimalloc`` + debug ``mimalloc`` + debug +=================================== ======================= ==================== ====================== ====================== Legend: @@ -417,8 +447,7 @@ Legend: * ``malloc``: system allocators from the standard C library, C functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`. * ``pymalloc``: :ref:`pymalloc memory allocator `. -* ``mimalloc``: :ref:`mimalloc memory allocator `. The pymalloc - allocator will be used if mimalloc support isn't available. +* ``mimalloc``: :ref:`mimalloc memory allocator `. * "+ debug": with :ref:`debug hooks on the Python memory allocators `. * "Debug build": :ref:`Python build in debug mode `. @@ -655,7 +684,11 @@ The pymalloc allocator Python has a *pymalloc* allocator optimized for small objects (smaller or equal to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" with a fixed size of either 256 KiB on 32-bit platforms or 1 MiB on 64-bit -platforms. It falls back to :c:func:`PyMem_RawMalloc` and +platforms. When Python is configured with :option:`--with-pymalloc-hugepages`, +the arena size on 64-bit platforms is increased to 2 MiB to match the huge page +size, and arena allocation will attempt to use huge pages (``MAP_HUGETLB`` on +Linux, ``MEM_LARGE_PAGES`` on Windows) with automatic fallback to regular pages. +It falls back to :c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. *pymalloc* is the :ref:`default allocator ` of the @@ -711,9 +744,27 @@ The mimalloc allocator .. versionadded:: 3.13 -Python supports the mimalloc allocator when the underlying platform support is available. -mimalloc "is a general purpose allocator with excellent performance characteristics. -Initially developed by Daan Leijen for the runtime systems of the Koka and Lean languages." +Python supports the `mimalloc `__ +allocator when the underlying platform support is available. +mimalloc is a general purpose allocator with excellent performance +characteristics, initially developed by Daan Leijen for the runtime systems +of the Koka and Lean languages. + +Unlike :ref:`pymalloc `, which is optimized for small objects (512 +bytes or fewer), mimalloc handles allocations of any size. + +In the :term:`free-threaded ` build, mimalloc is the default +and **required** allocator for the :c:macro:`PYMEM_DOMAIN_MEM` and +:c:macro:`PYMEM_DOMAIN_OBJ` domains. It cannot be disabled in free-threaded +builds. The free-threaded build uses per-thread mimalloc heaps, which allows +allocation and deallocation to proceed without locking in most cases. + +In the default (non-free-threaded) build, mimalloc is available but not the +default allocator. It can be selected at runtime using +:envvar:`PYTHONMALLOC`\ ``=mimalloc`` (or ``mimalloc_debug`` to include +:ref:`debug hooks `). It can be disabled at build time +using the :option:`--without-mimalloc` configure option, but this option +cannot be combined with :option:`--disable-gil`. tracemalloc C API ================= diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index f6038032805..e4ac8b57673 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -13,6 +13,12 @@ A :class:`memoryview` object exposes the C level :ref:`buffer interface any other object. +.. c:var:: PyTypeObject PyMemoryView_Type + + This instance of :c:type:`PyTypeObject` represents the Python memoryview + type. This is the same object as :class:`memoryview` in the Python layer. + + .. c:function:: PyObject *PyMemoryView_FromObject(PyObject *obj) Create a memoryview object from an object that provides the buffer interface. diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 29d5cd8d539..b67ca671a2a 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -3,17 +3,16 @@ .. _moduleobjects: Module Objects --------------- +============== .. index:: pair: object; module - .. c:var:: PyTypeObject PyModule_Type .. index:: single: ModuleType (in module types) This instance of :c:type:`PyTypeObject` represents the Python module type. This - is exposed to Python programs as ``types.ModuleType``. + is exposed to Python programs as :py:class:`types.ModuleType`. .. c:function:: int PyModule_Check(PyObject *p) @@ -71,6 +70,9 @@ Module Objects ``PyObject_*`` functions rather than directly manipulate a module's :attr:`~object.__dict__`. + The returned reference is borrowed from the module; it is valid until + the module is destroyed. + .. c:function:: PyObject* PyModule_GetNameObject(PyObject *module) @@ -90,18 +92,19 @@ Module Objects Similar to :c:func:`PyModule_GetNameObject` but return the name encoded to ``'utf-8'``. -.. c:function:: void* PyModule_GetState(PyObject *module) - - Return the "state" of the module, that is, a pointer to the block of memory - allocated at module creation time, or ``NULL``. See - :c:member:`PyModuleDef.m_size`. - + The returned buffer is only valid until the module is renamed or destroyed. + Note that Python code may rename a module by setting its :py:attr:`~module.__name__` + attribute. .. c:function:: PyModuleDef* PyModule_GetDef(PyObject *module) Return a pointer to the :c:type:`PyModuleDef` struct from which the module was created, or ``NULL`` if the module wasn't created from a definition. + On error, return ``NULL`` with an exception set. + Use :c:func:`PyErr_Occurred` to tell this case apart from a missing + :c:type:`!PyModuleDef`. + .. c:function:: PyObject* PyModule_GetFilenameObject(PyObject *module) @@ -122,215 +125,119 @@ Module Objects Similar to :c:func:`PyModule_GetFilenameObject` but return the filename encoded to 'utf-8'. + The returned buffer is only valid until the module's :py:attr:`~module.__file__` attribute + is reassigned or the module is destroyed. + .. deprecated:: 3.2 :c:func:`PyModule_GetFilename` raises :exc:`UnicodeEncodeError` on unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead. -.. _pymoduledef: +.. _pymoduledef_slot: -Module definitions ------------------- +Module definition +----------------- -The functions in the previous section work on any module object, including -modules imported from Python code. +Modules created using the C API are typically defined using an +array of :dfn:`slots`. +The slots provide a "description" of how a module should be created. -Modules defined using the C API typically use a *module definition*, -:c:type:`PyModuleDef` -- a statically allocated, constant “description" of -how a module should be created. +.. versionchanged:: 3.15 -The definition is usually used to define an extension's “main” module object -(see :ref:`extension-modules` for details). -It is also used to -:ref:`create extension modules dynamically `. + Previously, a :c:type:`PyModuleDef` struct was necessary to define modules. + The older way of defining modules is still available: consult either the + :ref:`pymoduledef` section or earlier versions of this documentation + if you plan to support earlier Python versions. -Unlike :c:func:`PyModule_New`, the definition allows management of -*module state* -- a piece of memory that is allocated and cleared together -with the module object. -Unlike the module's Python attributes, Python code cannot replace or delete -data stored in module state. +The slots array is usually used to define an extension module's “main” +module object (see :ref:`extension-modules` for details). +It can also be used to +:ref:`create extension modules dynamically `. -.. c:type:: PyModuleDef +Unless specified otherwise, the same slot ID may not be repeated +in an array of slots. - The module definition struct, which holds all information needed to create - a module object. - This structure must be statically allocated (or be otherwise guaranteed - to be valid while any modules created from it exist). - Usually, there is only one variable of this type for each extension module. - - .. c:member:: PyModuleDef_Base m_base - - Always initialize this member to :c:macro:`PyModuleDef_HEAD_INIT`. - - .. c:member:: const char *m_name - - Name for the new module. - - .. c:member:: const char *m_doc - - Docstring for the module; usually a docstring variable created with - :c:macro:`PyDoc_STRVAR` is used. - - .. c:member:: Py_ssize_t m_size - - Module state may be kept in a per-module memory area that can be - retrieved with :c:func:`PyModule_GetState`, rather than in static globals. - This makes modules safe for use in multiple sub-interpreters. - - This memory area is allocated based on *m_size* on module creation, - and freed when the module object is deallocated, after the - :c:member:`~PyModuleDef.m_free` function has been called, if present. - - Setting it to a non-negative value means that the module can be - re-initialized and specifies the additional amount of memory it requires - for its state. - - Setting ``m_size`` to ``-1`` means that the module does not support - sub-interpreters, because it has global state. - Negative ``m_size`` is only allowed when using - :ref:`legacy single-phase initialization ` - or when :ref:`creating modules dynamically `. - - See :PEP:`3121` for more details. - - .. c:member:: PyMethodDef* m_methods - - A pointer to a table of module-level functions, described by - :c:type:`PyMethodDef` values. Can be ``NULL`` if no functions are present. - - .. c:member:: PyModuleDef_Slot* m_slots - - An array of slot definitions for multi-phase initialization, terminated by - a ``{0, NULL}`` entry. - When using legacy single-phase initialization, *m_slots* must be ``NULL``. - - .. versionchanged:: 3.5 - - Prior to version 3.5, this member was always set to ``NULL``, - and was defined as: - - .. c:member:: inquiry m_reload - - .. c:member:: traverseproc m_traverse - - A traversal function to call during GC traversal of the module object, or - ``NULL`` if not needed. - - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. - - .. versionchanged:: 3.9 - No longer called before the module state is allocated. - - .. c:member:: inquiry m_clear - - A clear function to call during GC clearing of the module object, or - ``NULL`` if not needed. - - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. - - Like :c:member:`PyTypeObject.tp_clear`, this function is not *always* - called before a module is deallocated. For example, when reference - counting is enough to determine that an object is no longer used, - the cyclic garbage collector is not involved and - :c:member:`~PyModuleDef.m_free` is called directly. - - .. versionchanged:: 3.9 - No longer called before the module state is allocated. - - .. c:member:: freefunc m_free - - A function to call during deallocation of the module object, or ``NULL`` - if not needed. - - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. - - .. versionchanged:: 3.9 - No longer called before the module state is allocated. - - -Module slots -............ .. c:type:: PyModuleDef_Slot .. c:member:: int slot - A slot ID, chosen from the available values explained below. + A slot ID, chosen from the available ``Py_mod_*`` values explained below. + + An ID of 0 marks the end of a :c:type:`!PyModuleDef_Slot` array. .. c:member:: void* value Value of the slot, whose meaning depends on the slot ID. + The value may not be NULL. + To leave a slot out, omit the :c:type:`PyModuleDef_Slot` entry entirely. + .. versionadded:: 3.5 -The available slot types are: -.. c:macro:: Py_mod_create +Metadata slots +.............. - Specifies a function that is called to create the module object itself. - The *value* pointer of this slot must point to a function of the signature: +.. c:macro:: Py_mod_name - .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) - :no-index-entry: - :no-contents-entry: + :c:type:`Slot ID ` for the name of the new module, + as a NUL-terminated UTF8-encoded ``const char *``. - The function receives a :py:class:`~importlib.machinery.ModuleSpec` - instance, as defined in :PEP:`451`, and the module definition. - It should return a new module object, or set an error - and return ``NULL``. + Note that modules are typically created using a + :py:class:`~importlib.machinery.ModuleSpec`, and when they are, the + name from the spec will be used instead of :c:data:`!Py_mod_name`. + However, it is still recommended to include this slot for introspection + and debugging purposes. - This function should be kept minimal. In particular, it should not - call arbitrary Python code, as trying to import the same module again may - result in an infinite loop. + .. versionadded:: 3.15 - Multiple ``Py_mod_create`` slots may not be specified in one module - definition. + Use :c:member:`PyModuleDef.m_name` instead to support previous versions. - If ``Py_mod_create`` is not specified, the import machinery will create - a normal module object using :c:func:`PyModule_New`. The name is taken from - *spec*, not the definition, to allow extension modules to dynamically adjust - to their place in the module hierarchy and be imported under different - names through symlinks, all while sharing a single module definition. +.. c:macro:: Py_mod_doc - There is no requirement for the returned object to be an instance of - :c:type:`PyModule_Type`. Any type can be used, as long as it supports - setting and getting import-related attributes. - However, only ``PyModule_Type`` instances may be returned if the - ``PyModuleDef`` has non-``NULL`` ``m_traverse``, ``m_clear``, - ``m_free``; non-zero ``m_size``; or slots other than ``Py_mod_create``. + :c:type:`Slot ID ` for the docstring of the new + module, as a NUL-terminated UTF8-encoded ``const char *``. -.. c:macro:: Py_mod_exec + Usually it is set to a variable created with :c:macro:`PyDoc_STRVAR`. - Specifies a function that is called to *execute* the module. - This is equivalent to executing the code of a Python module: typically, - this function adds classes and constants to the module. - The signature of the function is: + .. versionadded:: 3.15 - .. c:function:: int exec_module(PyObject* module) - :no-index-entry: - :no-contents-entry: + Use :c:member:`PyModuleDef.m_doc` instead to support previous versions. - If multiple ``Py_mod_exec`` slots are specified, they are processed in the - order they appear in the *m_slots* array. + +Feature slots +............. + +.. c:macro:: Py_mod_abi + + :c:type:`Slot ID ` whose value points to + a :c:struct:`PyABIInfo` structure describing the ABI that + the extension is using. + + A suitable :c:struct:`!PyABIInfo` variable can be defined using the + :c:macro:`PyABIInfo_VAR` macro, as in: + + .. code-block:: c + + PyABIInfo_VAR(abi_info); + + static PyModuleDef_Slot mymodule_slots[] = { + {Py_mod_abi, &abi_info}, + ... + }; + + When creating a module, Python checks the value of this slot + using :c:func:`PyABIInfo_Check`. + + This slot is required, except for modules created from + :c:struct:`PyModuleDef`. + + .. versionadded:: 3.15 .. c:macro:: Py_mod_multiple_interpreters - Specifies one of the following values: + :c:type:`Slot ID ` whose value is one of: .. c:namespace:: NULL @@ -353,9 +260,6 @@ The available slot types are: This slot determines whether or not importing this module in a subinterpreter will fail. - Multiple ``Py_mod_multiple_interpreters`` slots may not be specified - in one module definition. - If ``Py_mod_multiple_interpreters`` is not specified, the import machinery defaults to ``Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED``. @@ -363,7 +267,7 @@ The available slot types are: .. c:macro:: Py_mod_gil - Specifies one of the following values: + :c:type:`Slot ID ` whose value is one of: .. c:namespace:: NULL @@ -381,45 +285,499 @@ The available slot types are: this module will cause the GIL to be automatically enabled. See :ref:`whatsnew313-free-threaded-cpython` for more detail. - Multiple ``Py_mod_gil`` slots may not be specified in one module definition. - If ``Py_mod_gil`` is not specified, the import machinery defaults to ``Py_MOD_GIL_USED``. .. versionadded:: 3.13 -.. c:macro:: Py_mod_abi - A pointer to a :c:struct:`PyABIInfo` structure that describes the ABI that - the extension is using. +Creation and initialization slots +................................. - When the module is loaded, the :c:struct:`!PyABIInfo` in this slot is checked - using :c:func:`PyABIInfo_Check`. +.. c:macro:: Py_mod_create - A suitable :c:struct:`!PyABIInfo` variable can be defined using the - :c:macro:`PyABIInfo_VAR` macro, as in: + :c:type:`Slot ID ` for a function that creates + the module object itself. + The function must have the signature: - .. code-block:: c + .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) + :no-index-entry: + :no-contents-entry: - PyABIInfo_VAR(abi_info); + The function will be called with: - static PyModuleDef_Slot mymodule_slots[] = { - {Py_mod_abi, &abi_info}, - ... - }; + - *spec*: a ``ModuleSpec``-like object, meaning that any attributes defined + for :py:class:`importlib.machinery.ModuleSpec` have matching semantics. + However, any of the attributes may be missing. + - *def*: ``NULL``, or the module definition if the module is created from one. + + The function should return a new module object, or set an error + and return ``NULL``. + + This function should be kept minimal. In particular, it should not + call arbitrary Python code, as trying to import the same module again may + result in an infinite loop. + + If ``Py_mod_create`` is not specified, the import machinery will create + a normal module object using :c:func:`PyModule_New`. The name is taken from + *spec*, not the definition, to allow extension modules to dynamically adjust + to their place in the module hierarchy and be imported under different + names through symlinks, all while sharing a single module definition. + + There is no requirement for the returned object to be an instance of + :c:type:`PyModule_Type`. + However, some slots may only be used with + :c:type:`!PyModule_Type` instances; in particular: + + - :c:macro:`Py_mod_exec`, + - :ref:`module state slots ` (``Py_mod_state_*``), + - :c:macro:`Py_mod_token`. + + .. versionadded:: 3.5 + + .. versionchanged:: 3.15 + + The *slots* argument may be a ``ModuleSpec``-like object, rather than + a true :py:class:`~importlib.machinery.ModuleSpec` instance. + Note that previous versions of CPython did not enforce this. + + The *def* argument may now be ``NULL``, since modules are not necessarily + made from definitions. + +.. c:macro:: Py_mod_exec + + :c:type:`Slot ID ` for a function that will + :dfn:`execute`, or initialize, the module. + This function does the equivalent to executing the code of a Python module: + typically, it adds classes and constants to the module. + The signature of the function is: + + .. c:function:: int exec_module(PyObject* module) + :no-index-entry: + :no-contents-entry: + + See the :ref:`capi-module-support-functions` section for some useful + functions to call. + + For backwards compatibility, the :c:type:`PyModuleDef.m_slots` array may + contain multiple :c:macro:`!Py_mod_exec` slots; these are processed in the + order they appear in the array. + Elsewhere (that is, in arguments to :c:func:`PyModule_FromSlotsAndSpec` + and in return values of :samp:`PyModExport_{}`), repeating the slot + is not allowed. + + .. versionadded:: 3.5 + + .. versionchanged:: 3.15 + + Repeated ``Py_mod_exec`` slots are disallowed, except in + :c:type:`PyModuleDef.m_slots`. + +.. c:macro:: Py_mod_methods + + :c:type:`Slot ID ` for a table of module-level + functions, as an array of :c:type:`PyMethodDef` values suitable as the + *functions* argument to :c:func:`PyModule_AddFunctions`. + + Like other slot IDs, a slots array may only contain one + :c:macro:`!Py_mod_methods` entry. + To add functions from multiple :c:type:`PyMethodDef` arrays, call + :c:func:`PyModule_AddFunctions` in the :c:macro:`Py_mod_exec` function. + + The table must be statically allocated (or otherwise guaranteed to outlive + the module object). + + .. versionadded:: 3.15 + + Use :c:member:`PyModuleDef.m_methods` instead to support previous versions. + +.. _ext-module-state: + +Module state +------------ + +Extension modules can have *module state* -- a +piece of memory that is allocated on module creation, +and freed when the module object is deallocated. +The module state is specified using :ref:`dedicated slots `. + +A typical use of module state is storing an exception type -- or indeed *any* +type object defined by the module -- + +Unlike the module's Python attributes, Python code cannot replace or delete +data stored in module state. + +Keeping per-module information in attributes and module state, rather than in +static globals, makes module objects *isolated* and safer for use in +multiple sub-interpreters. +It also helps Python do an orderly clean-up when it shuts down. + +Extensions that keep references to Python objects as part of module state must +implement :c:macro:`Py_mod_state_traverse` and :c:macro:`Py_mod_state_clear` +functions to avoid reference leaks. + +To retrieve the state from a given module, use the following functions: + +.. c:function:: void* PyModule_GetState(PyObject *module) + + Return the "state" of the module, that is, a pointer to the block of memory + allocated at module creation time, or ``NULL``. See + :c:macro:`Py_mod_state_size`. + + On error, return ``NULL`` with an exception set. + Use :c:func:`PyErr_Occurred` to tell this case apart from missing + module state. + + +.. c:function:: int PyModule_GetStateSize(PyObject *module, Py_ssize_t *result) + + Set *\*result* to the size of *module*'s state, as specified + using :c:macro:`Py_mod_state_size` (or :c:member:`PyModuleDef.m_size`), + and return 0. + + On error, set *\*result* to -1, and return -1 with an exception set. .. versionadded:: 3.15 -.. _moduledef-dynamic: + +.. _ext-module-state-slots: + +Slots for defining module state +............................... + +The following :c:member:`PyModuleDef_Slot.slot` IDs are available for +defining the module state. + +.. c:macro:: Py_mod_state_size + + :c:type:`Slot ID ` for the size of the module state, + in bytes. + + Setting the value to a non-negative value means that the module can be + re-initialized and specifies the additional amount of memory it requires + for its state. + + See :PEP:`3121` for more details. + + Use :c:func:`PyModule_GetStateSize` to retrieve the size of a given module. + + .. versionadded:: 3.15 + + Use :c:member:`PyModuleDef.m_size` instead to support previous versions. + +.. c:macro:: Py_mod_state_traverse + + :c:type:`Slot ID ` for a traversal function to call + during GC traversal of the module object. + + The signature of the function, and meanings of the arguments, + is similar as for :c:member:`PyTypeObject.tp_traverse`: + + .. c:function:: int traverse_module_state(PyObject *module, visitproc visit, void *arg) + :no-index-entry: + :no-contents-entry: + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if the state size + (:c:data:`Py_mod_state_size`) is greater than 0 and the module state + (as returned by :c:func:`PyModule_GetState`) is ``NULL``. + + .. versionadded:: 3.15 + + Use :c:member:`PyModuleDef.m_size` instead to support previous versions. + +.. c:macro:: Py_mod_state_clear + + :c:type:`Slot ID ` for a clear function to call + during GC clearing of the module object. + + The signature of the function is: + + .. c:function:: int clear_module_state(PyObject* module) + :no-index-entry: + :no-contents-entry: + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if the state size + (:c:data:`Py_mod_state_size`) is greater than 0 and the module state + (as returned by :c:func:`PyModule_GetState`) is ``NULL``. + + Like :c:member:`PyTypeObject.tp_clear`, this function is not *always* + called before a module is deallocated. For example, when reference + counting is enough to determine that an object is no longer used, + the cyclic garbage collector is not involved and + the :c:macro:`Py_mod_state_free` function is called directly. + + .. versionadded:: 3.15 + + Use :c:member:`PyModuleDef.m_clear` instead to support previous versions. + +.. c:macro:: Py_mod_state_free + + :c:type:`Slot ID ` for a function to call during + deallocation of the module object. + + The signature of the function is: + + .. c:function:: int free_module_state(PyObject* module) + :no-index-entry: + :no-contents-entry: + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if the state size + (:c:data:`Py_mod_state_size`) is greater than 0 and the module state + (as returned by :c:func:`PyModule_GetState`) is ``NULL``. + + .. versionadded:: 3.15 + + Use :c:member:`PyModuleDef.m_free` instead to support previous versions. + + +.. _ext-module-token: + +Module token +............ + +Each module may have an associated *token*: a pointer-sized value intended to +identify of the module state's memory layout. +This means that if you have a module object, but you are not sure if it +“belongs” to your extension, you can check using code like this: + +.. code-block:: c + + PyObject *module = + + void *module_token; + if (PyModule_GetToken(module, &module_token) < 0) { + return NULL; + } + if (module_token != your_token) { + PyErr_SetString(PyExc_ValueError, "unexpected module") + return NULL; + } + + // This module's state has the expected memory layout; it's safe to cast + struct my_state state = (struct my_state*)PyModule_GetState(module) + +A module's token -- and the *your_token* value to use in the above code -- is: + +- For modules created with :c:type:`PyModuleDef`: the address of that + :c:type:`PyModuleDef`; +- For modules defined with the :c:macro:`Py_mod_token` slot: the value + of that slot; +- For modules created from an ``PyModExport_*`` + :ref:`export hook `: the slots array that the export + hook returned (unless overridden with :c:macro:`Py_mod_token`). + +.. c:macro:: Py_mod_token + + :c:type:`Slot ID ` for the module token. + + If you use this slot to set the module token (rather than rely on the + default), you must ensure that: + + * The pointer outlives the class, so it's not reused for something else + while the class exists. + * It "belongs" to the extension module where the class lives, so it will not + clash with other extensions. + * If the token points to a :c:type:`PyModuleDef` struct, the module should + behave as if it was created from that :c:type:`PyModuleDef`. + In particular, the module state must have matching layout and semantics. + + Modules created from :c:type:`PyModuleDef` always use the address of + the :c:type:`PyModuleDef` as the token. + This means that :c:macro:`!Py_mod_token` cannot be used in + :c:member:`PyModuleDef.m_slots`. + + .. versionadded:: 3.15 + +.. c:function:: int PyModule_GetToken(PyObject *module, void** result) + + Set *\*result* to the module token for *module* and return 0. + + On error, set *\*result* to NULL, and return -1 with an exception set. + + .. versionadded:: 3.15 + +See also :c:func:`PyType_GetModuleByToken`. + + +.. _module-from-slots: Creating extension modules dynamically -------------------------------------- -The following functions may be used to create a module outside of an -extension's :ref:`initialization function `. -They are also used in -:ref:`single-phase initialization `. +The following functions may be used to create an extension module dynamically, +rather than from an extension's :ref:`export hook `. + +.. c:function:: PyObject *PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, PyObject *spec) + + Create a new module object, given an array of :ref:`slots ` + and the :py:class:`~importlib.machinery.ModuleSpec` *spec*. + + The *slots* argument must point to an array of :c:type:`PyModuleDef_Slot` + structures, terminated by an entry with slot ID of 0 + (typically written as ``{0}`` or ``{0, NULL}`` in C). + The array must include a :c:data:`Py_mod_abi` entry. + + The *spec* argument may be any ``ModuleSpec``-like object, as described + in :c:macro:`Py_mod_create` documentation. + Currently, the *spec* must have a ``name`` attribute. + + On success, return the new module. + On error, return ``NULL`` with an exception set. + + Note that this does not process the module's execution slot + (:c:data:`Py_mod_exec`). + Both :c:func:`!PyModule_FromSlotsAndSpec` and :c:func:`PyModule_Exec` + must be called to fully initialize a module. + (See also :ref:`multi-phase-initialization`.) + + The *slots* array only needs to be valid for the duration of the + :c:func:`!PyModule_FromSlotsAndSpec` call. + In particular, it may be heap-allocated. + + .. versionadded:: 3.15 + +.. c:function:: int PyModule_Exec(PyObject *module) + + Execute the :c:data:`Py_mod_exec` slot(s) of *module*. + + On success, return 0. + On error, return -1 with an exception set. + + For clarity: If *module* has no slots, for example if it uses + :ref:`legacy single-phase initialization `, + this function does nothing and returns 0. + + .. versionadded:: 3.15 + + + +.. _pymoduledef: + +Module definition struct +------------------------ + +Traditionally, extension modules were defined using a *module definition* +as the “description" of how a module should be created. +Rather than using an array of :ref:`slots ` directly, +the definition has dedicated members for most common functionality, +and allows additional slots as an extension mechanism. + +This way of defining modules is still available and there are no plans to +remove it. + +.. c:type:: PyModuleDef + + The module definition struct, which holds information needed to create + a module object. + + This structure must be statically allocated (or be otherwise guaranteed + to be valid while any modules created from it exist). + Usually, there is only one variable of this type for each extension module + defined this way. + + The struct, including all members, is part of the + :ref:`Stable ABI ` for non-free-threaded builds (``abi3``). + In the Stable ABI for free-threaded builds (``abi3t``), + this struct is opaque, and unusable in practice; see :ref:`pymoduledef_slot` + for a replacement. + + .. c:member:: PyModuleDef_Base m_base + + Always initialize this member to :c:macro:`PyModuleDef_HEAD_INIT`: + + .. c:namespace:: NULL + + .. c:type:: PyModuleDef_Base + + The type of :c:member:`!PyModuleDef.m_base`. + + The struct is part of the :ref:`Stable ABI ` for + non-free-threaded builds (``abi3``). + In the Stable ABI for Free-Threaded Builds + (``abi3t``), this struct is opaque, and unusable in practice. + + .. c:macro:: PyModuleDef_HEAD_INIT + + The required initial value for :c:member:`!PyModuleDef.m_base`. + + .. c:member:: const char *m_name + + Corresponds to the :c:macro:`Py_mod_name` slot. + + .. c:member:: const char *m_doc + + These members correspond to the :c:macro:`Py_mod_doc` slot. + Setting this to NULL is equivalent to omitting the slot. + + .. c:member:: Py_ssize_t m_size + + Corresponds to the :c:macro:`Py_mod_state_size` slot. + Setting this to zero is equivalent to omitting the slot. + + When using :ref:`legacy single-phase initialization ` + or when creating modules dynamically using :c:func:`PyModule_Create` + or :c:func:`PyModule_Create2`, :c:member:`!m_size` may be set to -1. + This indicates that the module does not support sub-interpreters, + because it has global state. + + .. c:member:: PyMethodDef *m_methods + + Corresponds to the :c:macro:`Py_mod_methods` slot. + Setting this to NULL is equivalent to omitting the slot. + + .. c:member:: PyModuleDef_Slot* m_slots + + An array of additional slots, terminated by a ``{0, NULL}`` entry. + + If the array contains slots corresponding to :c:type:`PyModuleDef` + members, the values must match. + For example, if you use :c:macro:`Py_mod_name` in :c:member:`!m_slots`, + :c:member:`PyModuleDef.m_name` must be set to the same pointer + (not just an equal string). + + .. versionchanged:: 3.5 + + Prior to version 3.5, this member was always set to ``NULL``, + and was defined as: + + .. c:member:: inquiry m_reload + + .. c:member:: traverseproc m_traverse + inquiry m_clear + freefunc m_free + + These members correspond to the :c:macro:`Py_mod_state_traverse`, + :c:macro:`Py_mod_state_clear`, and :c:macro:`Py_mod_state_free` slots, + respectively. + + Setting these members to NULL is equivalent to omitting the + corresponding slots. + + .. versionchanged:: 3.9 + + :c:member:`m_traverse`, :c:member:`m_clear` and :c:member:`m_free` + functions are no longer called before the module state is allocated. + + +.. c:var:: PyTypeObject PyModuleDef_Type + + The type of ``PyModuleDef`` objects. + + +.. _moduledef-dynamic: + +The following API can be used to create modules from a :c:type:`!PyModuleDef` +struct: .. c:function:: PyObject* PyModule_Create(PyModuleDef *def) @@ -482,26 +840,30 @@ They are also used in .. versionadded:: 3.5 .. c:macro:: PYTHON_API_VERSION + PYTHON_API_STRING - The C API version. Defined for backwards compatibility. + The C API version, as an integer (``1013``) and string (``"1013"``), respectively. + Defined for backwards compatibility. Currently, this constant is not updated in new Python versions, and is not useful for versioning. This may change in the future. .. c:macro:: PYTHON_ABI_VERSION + PYTHON_ABI_STRING - Defined as ``3`` for backwards compatibility. + Defined as ``3`` and ``"3"``, respectively, for backwards compatibility. Currently, this constant is not updated in new Python versions, and is not useful for versioning. This may change in the future. +.. _capi-module-support-functions: + Support functions ----------------- -The following functions are provided to help initialize a module -state. -They are intended for a module's execution slots (:c:data:`Py_mod_exec`), +The following functions are provided to help initialize a module object. +They are intended for a module's execution slot (:c:data:`Py_mod_exec`), the initialization function for legacy :ref:`single-phase initialization `, or code that creates modules dynamically. @@ -603,9 +965,7 @@ or code that creates modules dynamically. // PyModule_AddObject() stole a reference to obj: // Py_XDECREF(obj) is not needed here. - .. deprecated:: 3.13 - - :c:func:`PyModule_AddObject` is :term:`soft deprecated`. + .. soft-deprecated:: 3.13 .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) @@ -667,6 +1027,9 @@ or code that creates modules dynamically. :c:type:`PyMethodDef` arrays; in that case they should call this function directly. + The *functions* array must be statically allocated (or otherwise guaranteed + to outlive the module object). + .. versionadded:: 3.5 .. c:function:: int PyModule_SetDocString(PyObject *module, const char *docstring) @@ -676,6 +1039,9 @@ or code that creates modules dynamically. ``PyModuleDef`` (such as when using :ref:`multi-phase-initialization`, ``PyModule_Create``, or ``PyModule_FromDefAndSpec``). + Return ``0`` on success. + Return ``-1`` with an exception set on error. + .. versionadded:: 3.5 .. c:function:: int PyUnstable_Module_SetGIL(PyObject *module, void *gil) diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst index 7926148302a..4bfcb86abf5 100644 --- a/Doc/c-api/monitoring.rst +++ b/Doc/c-api/monitoring.rst @@ -136,7 +136,7 @@ Managing the Monitoring State ----------------------------- Monitoring states can be managed with the help of monitoring scopes. A scope -would typically correspond to a python function. +would typically correspond to a Python function. .. c:function:: int PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, const uint8_t *event_types, Py_ssize_t length) @@ -205,6 +205,4 @@ would typically correspond to a python function. .. versionadded:: 3.13 - .. deprecated:: 3.14 - - This function is :term:`soft deprecated`. + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 78599e704b1..eedeb180c6b 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -73,7 +73,7 @@ Object Protocol Flag to be used with multiple functions that print the object (like :c:func:`PyObject_Print` and :c:func:`PyFile_WriteObject`). - If passed, these function would use the :func:`str` of the object + If passed, these functions use the :func:`str` of the object instead of the :func:`repr`. @@ -85,6 +85,35 @@ Object Protocol instead of the :func:`repr`. +.. c:function:: void PyObject_Dump(PyObject *op) + + Dump an object *op* to ``stderr``. This should only be used for debugging. + + The output is intended to try dumping objects even after memory corruption: + + * Information is written starting with fields that are the least likely to + crash when accessed. + * This function can be called without an :term:`attached thread state`, but + it's not recommended to do so: it can cause deadlocks. + * An object that does not belong to the current interpreter may be dumped, + but this may also cause crashes or unintended behavior. + * Implement a heuristic to detect if the object memory has been freed. Don't + display the object contents in this case, only its memory address. + * The output format may change at any time. + + Example of output: + + .. code-block:: output + + object address : 0x7f80124702c0 + object refcount : 2 + object type : 0x9902e0 + object type name: str + object repr : 'abcdef' + + .. versionadded:: 3.15 + + .. c:function:: int PyObject_HasAttrWithError(PyObject *o, PyObject *attr_name) Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. @@ -201,7 +230,7 @@ Object Protocol This case can arise from forgetting ``NULL`` checks and would delete the attribute. - .. versionchanged:: next + .. versionchanged:: 3.15 Must not be called with NULL value if an exception is set. @@ -226,7 +255,7 @@ Object Protocol For more details, see :c:func:`PyUnicode_InternFromString`, which may be used internally to create a key object. - .. versionchanged:: next + .. versionchanged:: 3.15 Must not be called with NULL value if an exception is set. @@ -334,6 +363,8 @@ Object Protocol representation on success, ``NULL`` on failure. This is the equivalent of the Python expression ``repr(o)``. Called by the :func:`repr` built-in function. + If argument is ``NULL``, return the string ``''``. + .. versionchanged:: 3.4 This function now includes a debug assertion to help ensure that it does not silently discard an active exception. @@ -348,6 +379,8 @@ Object Protocol a string similar to that returned by :c:func:`PyObject_Repr` in Python 2. Called by the :func:`ascii` built-in function. + If argument is ``NULL``, return the string ``''``. + .. index:: string; PyObject_Str (C function) @@ -358,6 +391,8 @@ Object Protocol Python expression ``str(o)``. Called by the :func:`str` built-in function and, therefore, by the :func:`print` function. + If argument is ``NULL``, return the string ``''``. + .. versionchanged:: 3.4 This function now includes a debug assertion to help ensure that it does not silently discard an active exception. @@ -373,6 +408,8 @@ Object Protocol a TypeError is raised when *o* is an integer instead of a zero-initialized bytes object. + If argument is ``NULL``, return the :class:`bytes` object ``b''``. + .. c:function:: int PyObject_IsSubclass(PyObject *derived, PyObject *cls) @@ -600,7 +637,7 @@ Object Protocol Clear the managed dictionary of *obj*. - This function must only be called in a traverse function of the type which + This function must only be called in a clear function of the type which has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set. .. versionadded:: 3.13 @@ -682,10 +719,10 @@ Object Protocol :c:func:`PyUnstable_EnableTryIncRef` must have been called earlier on *obj* or this function may spuriously return ``0`` in the - :term:`free threading` build. + :term:`free-threaded build`. This function is logically equivalent to the following C code, except that - it behaves atomically in the :term:`free threading` build:: + it behaves atomically in the :term:`free-threaded build`:: if (Py_REFCNT(op) > 0) { Py_INCREF(op); @@ -762,13 +799,30 @@ Object Protocol On GIL-enabled builds, this function is equivalent to :c:expr:`Py_REFCNT(op) == 1`. - On a :term:`free threaded ` build, this checks if *op*'s + On a :term:`free-threaded build`, this checks if *op*'s :term:`reference count` is equal to one and additionally checks if *op* is only used by this thread. :c:expr:`Py_REFCNT(op) == 1` is **not** - thread-safe on free threaded builds; prefer this function. + thread-safe on free-threaded builds; prefer this function. The caller must hold an :term:`attached thread state`, despite the fact that this function doesn't call into the Python interpreter. This function cannot fail. .. versionadded:: 3.14 + +.. c:function:: int PyUnstable_SetImmortal(PyObject *op) + + Marks the object *op* :term:`immortal`. The argument should be uniquely referenced by + the calling thread. This is intended to be used for reducing reference counting contention + in the :term:`free-threaded build` for objects which are shared across threads. + + This is a one-way process: objects can only be made immortal; they cannot be + made mortal once again. Immortal objects do not participate in reference counting + and will never be garbage collected. If the object is GC-tracked, it is untracked. + + This function is intended to be used soon after *op* is created, by the code that + creates it, such as in the object's :c:member:`~PyTypeObject.tp_new` slot. + Returns 1 if the object was made immortal and returns 0 if it was not. + This function cannot fail. + + .. versionadded:: 3.15 diff --git a/Doc/c-api/perfmaps.rst b/Doc/c-api/perfmaps.rst index 76a1e9f528d..bd05e628faa 100644 --- a/Doc/c-api/perfmaps.rst +++ b/Doc/c-api/perfmaps.rst @@ -31,7 +31,7 @@ Note that holding an :term:`attached thread state` is not required for these API or ``-2`` on failure to create a lock. Check ``errno`` for more information about the cause of a failure. -.. c:function:: int PyUnstable_WritePerfMapEntry(const void *code_addr, unsigned int code_size, const char *entry_name) +.. c:function:: int PyUnstable_WritePerfMapEntry(const void *code_addr, size_t code_size, const char *entry_name) Write one single entry to the ``/tmp/perf-$pid.map`` file. This function is thread safe. Here is what an example entry looks like:: diff --git a/Doc/c-api/picklebuffer.rst b/Doc/c-api/picklebuffer.rst new file mode 100644 index 00000000000..9e2d92341b0 --- /dev/null +++ b/Doc/c-api/picklebuffer.rst @@ -0,0 +1,59 @@ +.. highlight:: c + +.. _picklebuffer-objects: + +.. index:: + pair: object; PickleBuffer + +Pickle buffer objects +--------------------- + +.. versionadded:: 3.8 + +A :class:`pickle.PickleBuffer` object wraps a :ref:`buffer-providing object +` for out-of-band data transfer with the :mod:`pickle` module. + + +.. c:var:: PyTypeObject PyPickleBuffer_Type + + This instance of :c:type:`PyTypeObject` represents the Python pickle buffer type. + This is the same object as :class:`pickle.PickleBuffer` in the Python layer. + + +.. c:function:: int PyPickleBuffer_Check(PyObject *op) + + Return true if *op* is a pickle buffer instance. + This function always succeeds. + + +.. c:function:: PyObject *PyPickleBuffer_FromObject(PyObject *obj) + + Create a pickle buffer from the object *obj*. + + This function will fail if *obj* doesn't support the :ref:`buffer protocol `. + + On success, return a new pickle buffer instance. + On failure, set an exception and return ``NULL``. + + Analogous to calling :class:`pickle.PickleBuffer` with *obj* in Python. + + +.. c:function:: const Py_buffer *PyPickleBuffer_GetBuffer(PyObject *picklebuf) + + Get a pointer to the underlying :c:type:`Py_buffer` that the pickle buffer wraps. + + The returned pointer is valid as long as *picklebuf* is alive and has not been + released. The caller must not modify or free the returned :c:type:`Py_buffer`. + If the pickle buffer has been released, raise :exc:`ValueError`. + + On success, return a pointer to the buffer view. + On failure, set an exception and return ``NULL``. + + +.. c:function:: int PyPickleBuffer_Release(PyObject *picklebuf) + + Release the underlying buffer held by the pickle buffer. + + Return ``0`` on success. On failure, set an exception and return ``-1``. + + Analogous to calling :meth:`pickle.PickleBuffer.release` in Python. diff --git a/Doc/c-api/profiling.rst b/Doc/c-api/profiling.rst new file mode 100644 index 00000000000..0200f2eac6d --- /dev/null +++ b/Doc/c-api/profiling.rst @@ -0,0 +1,239 @@ +.. highlight:: c + +.. _profiling: + +Profiling and tracing +===================== + +The Python interpreter provides some low-level support for attaching profiling +and execution tracing facilities. These are used for profiling, debugging, and +coverage analysis tools. + +This C interface allows the profiling or tracing code to avoid the overhead of +calling through Python-level callable objects, making a direct C function call +instead. The essential attributes of the facility have not changed; the +interface allows trace functions to be installed per-thread, and the basic +events reported to the trace function are the same as had been reported to the +Python-level trace functions in previous versions. + + +.. c:type:: int (*Py_tracefunc)(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) + + The type of the trace function registered using :c:func:`PyEval_SetProfile` and + :c:func:`PyEval_SetTrace`. The first parameter is the object passed to the + registration function as *obj*, *frame* is the frame object to which the event + pertains, *what* is one of the constants :c:data:`PyTrace_CALL`, + :c:data:`PyTrace_EXCEPTION`, :c:data:`PyTrace_LINE`, :c:data:`PyTrace_RETURN`, + :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION`, :c:data:`PyTrace_C_RETURN`, + or :c:data:`PyTrace_OPCODE`, and *arg* depends on the value of *what*: + + +-------------------------------+----------------------------------------+ + | Value of *what* | Meaning of *arg* | + +===============================+========================================+ + | :c:data:`PyTrace_CALL` | Always :c:data:`Py_None`. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_EXCEPTION` | Exception information as returned by | + | | :func:`sys.exc_info`. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_LINE` | Always :c:data:`Py_None`. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_RETURN` | Value being returned to the caller, | + | | or ``NULL`` if caused by an exception. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_C_CALL` | Function object being called. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_C_EXCEPTION` | Function object being called. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_C_RETURN` | Function object being called. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_OPCODE` | Always :c:data:`Py_None`. | + +-------------------------------+----------------------------------------+ + +.. c:var:: int PyTrace_CALL + + The value of the *what* parameter to a :c:type:`Py_tracefunc` function when a new + call to a function or method is being reported, or a new entry into a generator. + Note that the creation of the iterator for a generator function is not reported + as there is no control transfer to the Python bytecode in the corresponding + frame. + + +.. c:var:: int PyTrace_EXCEPTION + + The value of the *what* parameter to a :c:type:`Py_tracefunc` function when an + exception has been raised. The callback function is called with this value for + *what* when after any bytecode is processed after which the exception becomes + set within the frame being executed. The effect of this is that as exception + propagation causes the Python stack to unwind, the callback is called upon + return to each frame as the exception propagates. Only trace functions receive + these events; they are not needed by the profiler. + + +.. c:var:: int PyTrace_LINE + + The value passed as the *what* parameter to a :c:type:`Py_tracefunc` function + (but not a profiling function) when a line-number event is being reported. + It may be disabled for a frame by setting :attr:`~frame.f_trace_lines` to + *0* on that frame. + + +.. c:var:: int PyTrace_RETURN + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a + call is about to return. + + +.. c:var:: int PyTrace_C_CALL + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C + function is about to be called. + + +.. c:var:: int PyTrace_C_EXCEPTION + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C + function has raised an exception. + + +.. c:var:: int PyTrace_C_RETURN + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C + function has returned. + + +.. c:var:: int PyTrace_OPCODE + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions (but not + profiling functions) when a new opcode is about to be executed. This event is + not emitted by default: it must be explicitly requested by setting + :attr:`~frame.f_trace_opcodes` to *1* on the frame. + + +.. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) + + Set the profiler function to *func*. The *obj* parameter is passed to the + function as its first parameter, and may be any Python object, or ``NULL``. If + the profile function needs to maintain state, using a different value for *obj* + for each thread provides a convenient and thread-safe place to store it. The + profile function is called for all monitored events except :c:data:`PyTrace_LINE` + :c:data:`PyTrace_OPCODE` and :c:data:`PyTrace_EXCEPTION`. + + See also the :func:`sys.setprofile` function. + + The caller must have an :term:`attached thread state`. + + +.. c:function:: void PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *obj) + + Like :c:func:`PyEval_SetProfile` but sets the profile function in all running threads + belonging to the current interpreter instead of the setting it only on the current thread. + + The caller must have an :term:`attached thread state`. + + As :c:func:`PyEval_SetProfile`, this function ignores any exceptions raised while + setting the profile functions in all threads. + +.. versionadded:: 3.12 + + +.. c:function:: void PyEval_SetTrace(Py_tracefunc func, PyObject *obj) + + Set the tracing function to *func*. This is similar to + :c:func:`PyEval_SetProfile`, except the tracing function does receive line-number + events and per-opcode events, but does not receive any event related to C function + objects being called. Any trace function registered using :c:func:`PyEval_SetTrace` + will not receive :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION` or + :c:data:`PyTrace_C_RETURN` as a value for the *what* parameter. + + See also the :func:`sys.settrace` function. + + The caller must have an :term:`attached thread state`. + + +.. c:function:: void PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *obj) + + Like :c:func:`PyEval_SetTrace` but sets the tracing function in all running threads + belonging to the current interpreter instead of the setting it only on the current thread. + + The caller must have an :term:`attached thread state`. + + As :c:func:`PyEval_SetTrace`, this function ignores any exceptions raised while + setting the trace functions in all threads. + +.. versionadded:: 3.12 + + +Reference tracing +================= + +.. versionadded:: 3.13 + + +.. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data) + + The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`. + The first parameter is a Python object that has been just created (when **event** + is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event** + is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer + that was provided when :c:func:`PyRefTracer_SetTracer` was called. + + If a new tracing function is registered replacing the current one, a call to the + trace function will be made with the object set to **NULL** and **event** set to + :c:data:`PyRefTracer_TRACKER_REMOVED`. This will happen just before the new + function is registered. + +.. versionadded:: 3.13 + + +.. c:var:: int PyRefTracer_CREATE + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been created. + + +.. c:var:: int PyRefTracer_DESTROY + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been destroyed. + + +.. c:var:: int PyRefTracer_TRACKER_REMOVED + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when the + current tracer is about to be replaced by a new one. + + .. versionadded:: 3.14 + + +.. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) + + Register a reference tracer function. The function will be called when a new + Python object has been created or when an object is going to be destroyed. If + **data** is provided it must be an opaque pointer that will be provided when + the tracer function is called. Return ``0`` on success. Set an exception and + return ``-1`` on error. + + Note that tracer functions **must not** create Python objects inside or + otherwise the call will be re-entrant. The tracer also **must not** clear + any existing exception or set an exception. A :term:`thread state` will be active + every time the tracer function is called. + + There must be an :term:`attached thread state` when calling this function. + + If another tracer function was already registered, the old function will be + called with **event** set to :c:data:`PyRefTracer_TRACKER_REMOVED` just before + the new function is registered. + +.. versionadded:: 3.13 + + +.. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data) + + Get the registered reference tracer function and the value of the opaque data + pointer that was registered when :c:func:`PyRefTracer_SetTracer` was called. + If no tracer was registered this function will return NULL and will set the + **data** pointer to NULL. + + There must be an :term:`attached thread state` when calling this function. + +.. versionadded:: 3.13 diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 57a0728d4e9..4d56a92bf2a 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -25,7 +25,7 @@ of Python objects. .. note:: - On :term:`free threaded ` builds of Python, returning 1 + On :term:`free-threaded builds ` of Python, returning 1 isn't sufficient to determine if it's safe to treat *o* as having no access by other threads. Use :c:func:`PyUnstable_Object_IsUniquelyReferenced` for that instead. diff --git a/Doc/c-api/sentinel.rst b/Doc/c-api/sentinel.rst new file mode 100644 index 00000000000..710ded56e2a --- /dev/null +++ b/Doc/c-api/sentinel.rst @@ -0,0 +1,35 @@ +.. highlight:: c + +.. _sentinelobjects: + +Sentinel objects +---------------- + +.. c:var:: PyTypeObject PySentinel_Type + + This instance of :c:type:`PyTypeObject` represents the Python + :class:`sentinel` type. This is the same object as :class:`sentinel`. + + .. versionadded:: next + +.. c:function:: int PySentinel_Check(PyObject *o) + + Return true if *o* is a :class:`sentinel` object. The :class:`sentinel` type + does not allow subclasses, so this check is exact. + + .. versionadded:: next + +.. c:function:: PyObject* PySentinel_New(const char *name, const char *module_name) + + Return a new :class:`sentinel` object with :attr:`~sentinel.__name__` set to + *name* and :attr:`~sentinel.__module__` set to *module_name*. + *name* must not be ``NULL``. If *module_name* is ``NULL``, :attr:`~sentinel.__module__` + is set to ``None``. + Return ``NULL`` with an exception set on failure. + + For pickling to work, *module_name* must be the name of an importable + module, and the sentinel must be accessible from that module under a + path matching *name*. Pickle treats *name* as a global variable name + in *module_name* (see :meth:`object.__reduce__`). + + .. versionadded:: next diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index df5bf6b64a9..6bae8f25ad7 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -109,9 +109,8 @@ Sequence Protocol Alias for :c:func:`PySequence_Contains`. - .. deprecated:: 3.14 - The function is :term:`soft deprecated` and should no longer be used to - write new code. + .. soft-deprecated:: 3.14 + The function should no longer be used to write new code. .. c:function:: Py_ssize_t PySequence_Index(PyObject *o, PyObject *value) diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index cba823aa027..db537aff2e6 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -5,9 +5,6 @@ Set Objects ----------- -.. sectionauthor:: Raymond D. Hettinger - - .. index:: pair: object; set pair: object; frozenset @@ -92,6 +89,11 @@ the constructor functions work with any iterable Python object. actually iterable. The constructor is also useful for copying a set (``c=set(s)``). + .. note:: + + The operation is atomic on :term:`free threading ` + when *iterable* is a :class:`set`, :class:`frozenset`, :class:`dict` or :class:`frozendict`. + .. c:function:: PyObject* PyFrozenSet_New(PyObject *iterable) @@ -100,6 +102,11 @@ the constructor functions work with any iterable Python object. set on success or ``NULL`` on failure. Raise :exc:`TypeError` if *iterable* is not actually iterable. + .. note:: + + The operation is atomic on :term:`free threading ` + when *iterable* is a :class:`set`, :class:`frozenset`, :class:`dict` or :class:`frozendict`. + The following functions and macros are available for instances of :class:`set` or :class:`frozenset` or instances of their subtypes. @@ -127,6 +134,10 @@ or :class:`frozenset` or instances of their subtypes. the *key* is unhashable. Raise :exc:`SystemError` if *anyset* is not a :class:`set`, :class:`frozenset`, or an instance of a subtype. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. .. c:function:: int PySet_Add(PyObject *set, PyObject *key) @@ -138,6 +149,12 @@ or :class:`frozenset` or instances of their subtypes. :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + + The following functions are available for instances of :class:`set` or its subtypes but not for instances of :class:`frozenset` or its subtypes. @@ -147,11 +164,16 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. Return ``1`` if found and removed, ``0`` if not found (no action taken), and ``-1`` if an error is encountered. Does not raise :exc:`KeyError` for missing keys. Raise a - :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~frozenset.discard` + :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~set.discard` method, this function does not automatically convert unhashable sets into temporary frozensets. Raise :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. c:function:: PyObject* PySet_Pop(PyObject *set) @@ -166,3 +188,28 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. Empty an existing set of all elements. Return ``0`` on success. Return ``-1`` and raise :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. + + .. note:: + + In the :term:`free-threaded build`, the set is emptied before its entries + are cleared, so other threads will observe an empty set rather than + intermediate states. + + +Deprecated API +^^^^^^^^^^^^^^ + +.. c:macro:: PySet_MINSIZE + + A constant representing the size of an internal + preallocated table inside :c:type:`PySetObject` instances. + + This is documented solely for completeness, as there are no guarantees + that a given version of CPython uses preallocated tables with a fixed + size. + In code that does not deal with unstable set internals, + :c:macro:`!PySet_MINSIZE` can be replaced with a small constant like ``8``. + + If looking for the size of a set, use :c:func:`PySet_Size` instead. + + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 8fed4b69b25..fe92f72f8eb 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -51,135 +51,212 @@ It is generally intended for specialized, low-level tools like debuggers. Projects that use this API are expected to follow CPython development and spend extra effort adjusting to changes. +.. _stable-abi: .. _stable-application-binary-interface: -Stable Application Binary Interface -=================================== +Stable Application Binary Interfaces +==================================== -For simplicity, this document talks about *extensions*, but the Limited API -and Stable ABI work the same way for all uses of the API – for example, -embedding Python. - -.. _limited-c-api: - -Limited C API -------------- - -Python 3.2 introduced the *Limited API*, a subset of Python's C API. -Extensions that only use the Limited API can be -compiled once and be loaded on multiple versions of Python. -Contents of the Limited API are :ref:`listed below `. - -.. c:macro:: Py_LIMITED_API - - Define this macro before including ``Python.h`` to opt in to only use - the Limited API, and to select the Limited API version. - - Define ``Py_LIMITED_API`` to the value of :c:macro:`PY_VERSION_HEX` - corresponding to the lowest Python version your extension supports. - The extension will be ABI-compatible with all Python 3 releases - from the specified one onward, and can use Limited API introduced up to that - version. - - Rather than using the ``PY_VERSION_HEX`` macro directly, hardcode a minimum - minor version (e.g. ``0x030A0000`` for Python 3.10) for stability when - compiling with future Python versions. - - You can also define ``Py_LIMITED_API`` to ``3``. This works the same as - ``0x03020000`` (Python 3.2, the version that introduced Limited API). - - -.. _stable-abi: - -Stable ABI ----------- - -To enable this, Python provides a *Stable ABI*: a set of symbols that will -remain ABI-compatible across Python 3.x versions. +Python's :dfn:`Stable ABI` allows extensions to be compatible with multiple +versions of Python, without recompilation. .. note:: - The Stable ABI prevents ABI issues, like linker errors due to missing - symbols or data corruption due to changes in structure layouts or function - signatures. - However, other changes in Python can change the *behavior* of extensions. - See Python's Backwards Compatibility Policy (:pep:`387`) for details. + For simplicity, this document talks about *extensions*, but Stable ABI + works the same way for all uses of the API – for example, embedding Python. -The Stable ABI contains symbols exposed in the :ref:`Limited API -`, but also other ones – for example, functions necessary to -support older versions of the Limited API. +There are two Stable ABIs: -On Windows, extensions that use the Stable ABI should be linked against +- ``abi3``, introduced in Python 3.2, is compatible with + **non**-:term:`free-threaded ` builds of CPython. + +- ``abi3t``, introduced in Python 3.15, is compatible with + :term:`free-threaded ` builds of CPython. + It has stricter API limitations than ``abi3``. + + .. versionadded:: next + + ``abi3t`` was added in :pep:`803` + +It is possible for an extension to be compiled for *both* ``abi3`` and +``abi3t`` at the same time; the result will be compatible with +both free-threaded and non-free-threaded builds of Python. +Currently, this has no downsides compared to compiling for ``abi3t`` only. + +Each Stable ABI is versioned using the first two numbers of the Python version. +For example, Stable ABI 3.14 corresponds to Python 3.14. +An extension compiled for Stable ABI 3.x is ABI-compatible with Python 3.x +and above. + +Extensions that target a stable ABI must only use a limited subset of +the C API. This subset is known as the :dfn:`Limited API`; its contents +are :ref:`listed below `. + +On Windows, extensions that use a Stable ABI should be linked against ``python3.dll`` rather than a version-specific library such as ``python39.dll``. +This library only exposes the relevant symbols. On some platforms, Python will look for and load shared library files named -with the ``abi3`` tag (e.g. ``mymodule.abi3.so``). -It does not check if such extensions conform to a Stable ABI. -The user (or their packaging tools) need to ensure that, for example, -extensions built with the 3.10+ Limited API are not installed for lower +with the ``abi3`` or ``abi3t`` tag (for example, ``mymodule.abi3.so``). +:term:`Free-threaded ` interpreters only recognize the +``abi3t`` tag, while non-free-threaded ones will prefer ``abi3`` but fall back +to ``abi3t``. +Thus, extensions compatible with both ABIs should use the ``abi3t`` tag. + +Python does not necessarily check that extensions it loads +have compatible ABI. +Extension authors are encouraged to add a check using the :c:macro:`Py_mod_abi` +slot or the :c:func:`PyABIInfo_Check` function, but the user +(or their packaging tool) is ultimately responsible for ensuring that, +for example, extensions built for Stable ABI 3.10 are not installed for lower versions of Python. -All functions in the Stable ABI are present as functions in Python's shared -library, not solely as macros. This makes them usable from languages that don't -use the C preprocessor. +All functions in Stable ABI are present as functions in Python's shared +library, not solely as macros. +This makes them usable are usable from languages that don't use the C +preprocessor, including Python's :py:mod:`ctypes`. -Limited API Scope and Performance ---------------------------------- +.. _abi3-compiling: -The goal for the Limited API is to allow everything that is possible with the +Compiling for Stable ABI +------------------------ + +.. note:: + + Build tools (such as, for example, meson-python, scikit-build-core, + or Setuptools) often have a mechanism for setting macros and synchronizing + them with extension filenames and other metadata. + Prefer using such a mechanism, if it exists, over defining the + macros manually. + + The rest of this section is mainly relevant for tool authors, and for + people who compile extensions manually. + + .. seealso:: `list of recommended tools`_ in the Python Packaging User Guide + + .. _list of recommended tools: https://packaging.python.org/en/latest/guides/tool-recommendations/#build-backends-for-extension-modules + +To compile for a Stable ABI, define one or both of the following macros +to the lowest Python version your extension should support, in +:c:macro:`Py_PACK_VERSION` format. +Typically, you should choose a specific value rather than the version of +the Python headers you are compiling against. + +The macros must be defined before including ``Python.h``. +Since :c:macro:`Py_PACK_VERSION` is not available at this point, you +will need to use the numeric value directly. +For reference, the values for a few recent Python versions are: + +.. version-hex-cheatsheet:: + +When one of the macros is defined, ``Python.h`` will only expose API that is +compatible with the given Stable ABI -- that is, the +:ref:`Limited API ` plus some definitions that need to be +visible to the compiler but should not be used directly. +When both are defined, ``Python.h`` will only expose API compatible with +both Stable ABIs. + +.. c:macro:: Py_LIMITED_API + + Target ``abi3``, that is, + non-:term:`free-threaded builds ` of CPython. + See :ref:`above ` for common information. + +.. c:macro:: Py_TARGET_ABI3T + + Target ``abi3t``, that is, + :term:`free-threaded builds ` of CPython. + See :ref:`above ` for common information. + + .. versionadded:: next + +Both macros specify a target ABI; the different naming style is due to +backwards compatibility. + +.. admonition:: Historical note + + You can also define ``Py_LIMITED_API`` as ``3``. This works the same as + ``0x03020000`` (Python 3.2, the version that introduced Stable ABI). + +When both are defined, ``Python.h`` may, or may not, redefine +:c:macro:`!Py_LIMITED_API` to match :c:macro:`!Py_TARGET_ABI3T`. + +On a :term:`free-threaded build` -- that is, when +:c:macro:`Py_GIL_DISABLED` is defined -- :c:macro:`!Py_TARGET_ABI3T` +defaults to the value of :c:macro:`!Py_LIMITED_API`. +This means that there are two ways to build for both ``abi3`` and ``abi3t``: + +- define both :c:macro:`!Py_LIMITED_API` and :c:macro:`!Py_TARGET_ABI3T`, or +- define only :c:macro:`!Py_LIMITED_API` and: + + - on Windows, define :c:macro:`!Py_GIL_DISABLED`; + - on other systems, use the headers of free-threaded build of Python. + + +.. _limited-api-scope-and-performance: + +Stable ABI Scope and Performance +-------------------------------- + +The goal for Stable ABI is to allow everything that is possible with the full C API, but possibly with a performance penalty. +Generally, compatibility with Stable ABI will require some changes to an +extension's source code. -For example, while :c:func:`PyList_GetItem` is available, its “unsafe” macro +For example, while :c:func:`PyList_GetItem` is available, its "unsafe" macro variant :c:func:`PyList_GET_ITEM` is not. The macro can be faster because it can rely on version-specific implementation details of the list object. -Without ``Py_LIMITED_API`` defined, some C API functions are inlined or -replaced by macros. -Defining ``Py_LIMITED_API`` disables this inlining, allowing stability as +For another example, when *not* compiling for Stable ABI, some C API +functions are inlined or replaced by macros. +Compiling for Stable ABI disables this inlining, allowing stability as Python's data structures are improved, but possibly reducing performance. -By leaving out the ``Py_LIMITED_API`` definition, it is possible to compile -a Limited API extension with a version-specific ABI. This can improve -performance for that Python version, but will limit compatibility. -Compiling with ``Py_LIMITED_API`` will then yield an extension that can be -distributed where a version-specific one is not available – for example, -for prereleases of an upcoming Python version. +By leaving out the :c:macro:`!Py_LIMITED_API` or :c:macro:`!Py_TARGET_ABI3T` +definition, it is possible to compile Stable-ABI-compatible source +for a version-specific ABI. +A potentially faster version-specific extension can then be distributed +alongside a version compiled for Stable ABI -- a slower but more compatible +fallback. -Limited API Caveats -------------------- +.. _limited-api-caveats: -Note that compiling with ``Py_LIMITED_API`` is *not* a complete guarantee that -code conforms to the :ref:`Limited API ` or the :ref:`Stable ABI -`. ``Py_LIMITED_API`` only covers definitions, but an API also -includes other issues, such as expected semantics. +Stable ABI Caveats +------------------ -One issue that ``Py_LIMITED_API`` does not guard against is calling a function -with arguments that are invalid in a lower Python version. +Note that compiling for Stable ABI is *not* a complete guarantee that code will +be compatible with the expected Python versions. +Stable ABI prevents *ABI* issues, like linker errors due to missing +symbols or data corruption due to changes in structure layouts or function +signatures. +However, other changes in Python can change the *behavior* of extensions. + +One issue that the :c:macro:`Py_TARGET_ABI3T` and :c:macro:`Py_LIMITED_API` +macros do not guard against is calling a function with arguments that are +invalid in a lower Python version. For example, consider a function that starts accepting ``NULL`` for an argument. In Python 3.9, ``NULL`` now selects a default behavior, but in Python 3.8, the argument will be used directly, causing a ``NULL`` dereference and crash. A similar argument works for fields of structs. -Another issue is that some struct fields are currently not hidden when -``Py_LIMITED_API`` is defined, even though they're part of the Limited API. - For these reasons, we recommend testing an extension with *all* minor Python -versions it supports, and preferably to build with the *lowest* such version. +versions it supports. We also recommend reviewing documentation of all used API to check if it is explicitly part of the Limited API. Even with ``Py_LIMITED_API`` defined, a few private declarations are exposed for technical reasons (or even unintentionally, as bugs). -Also note that the Limited API is not necessarily stable: compiling with -``Py_LIMITED_API`` with Python 3.8 means that the extension will -run with Python 3.12, but it will not necessarily *compile* with Python 3.12. -In particular, parts of the Limited API may be deprecated and removed, -provided that the Stable ABI stays stable. +Also note that while compiling with ``Py_LIMITED_API`` 3.8 means that the +extension should *load* on Python 3.12, and *compile* with Python 3.12, +the same source will not necessarily compile with ``Py_LIMITED_API`` +set to 3.12. +In general, parts of the Limited API may be deprecated and removed, +provided that Stable ABI stays stable. .. _stable-abi-platform: @@ -189,12 +266,12 @@ Platform Considerations ABI stability depends not only on Python, but also on the compiler used, lower-level libraries and compiler options. For the purposes of -the :ref:`Stable ABI `, these details define a “platform”. They +the :ref:`Stable ABIs `, these details define a “platform”. They usually depend on the OS type and processor architecture It is the responsibility of each particular distributor of Python to ensure that all Python versions on a particular platform are built -in a way that does not break the Stable ABI. +in a way that does not break the Stable ABIs, or the version-specific ABIs. This is the case with Windows and macOS releases from ``python.org`` and many third-party distributors. @@ -202,7 +279,7 @@ third-party distributors. ABI Checking ============ -.. versionadded:: next +.. versionadded:: 3.15 Python includes a rudimentary check for ABI compatibility. @@ -249,7 +326,7 @@ The full API is described below for advanced use cases. may lead to crashes. In particular, it is not safe to examine the raised exception. - .. versionadded:: next + .. versionadded:: 3.15 .. c:macro:: PyABIInfo_VAR(NAME) @@ -266,7 +343,7 @@ The full API is described below for advanced use cases. PyABIInfo_DEFAULT_ABI_VERSION } - .. versionadded:: next + .. versionadded:: 3.15 .. c:type:: PyABIInfo @@ -279,7 +356,7 @@ The full API is described below for advanced use cases. .. c:member:: uint8_t abiinfo_minor_version - The major version of :c:struct:`PyABIInfo`. + The minor version of :c:struct:`PyABIInfo`. Must be set to ``0``; larger values are reserved for backwards-compatible future versions of :c:struct:`!PyABIInfo`. @@ -294,7 +371,7 @@ The full API is described below for advanced use cases. Default flags, based on current values of macros such as :c:macro:`Py_LIMITED_API` and :c:macro:`Py_GIL_DISABLED`. - Alternately, the field can be set to to the following flags, combined + Alternately, the field can be set to the following flags, combined by bitwise OR. Unused bits must be set to zero. @@ -302,7 +379,7 @@ The full API is described below for advanced use cases. .. c:macro:: PyABIInfo_STABLE - Specifies that the stable ABI is used. + Specifies that Stable ABI is used. .. c:macro:: PyABIInfo_INTERNAL @@ -313,15 +390,22 @@ The full API is described below for advanced use cases. .. c:macro:: PyABIInfo_FREETHREADED - Specifies ABI compatible with free-threading builds of CPython. + Specifies ABI compatible with :term:`free-threaded builds + ` of CPython. (That is, ones compiled with :option:`--disable-gil`; with ``t`` in :py:data:`sys.abiflags`) .. c:macro:: PyABIInfo_GIL - Specifies ABI compatible with non-free-threading builds of CPython + Specifies ABI compatible with non-free-threaded builds of CPython (ones compiled *without* :option:`--disable-gil`). + .. c:macro:: PyABIInfo_FREETHREADING_AGNOSTIC + + Specifies ABI compatible with both free-threaded and + non-free-threaded builds of CPython, that is, both + ``abi3`` and ``abi3t``. + .. c:member:: uint32_t build_version The version of the Python headers used to build the code, in the format @@ -335,10 +419,11 @@ The full API is described below for advanced use cases. The ABI version. - For the Stable ABI, this field should be the value of - :c:macro:`Py_LIMITED_API` - (except if :c:macro:`Py_LIMITED_API` is ``3``; use - :c:expr:`Py_PACK_VERSION(3, 2)` in that case). + For Stable ABI, this field should be the value of + :c:macro:`Py_LIMITED_API` or :c:macro:`Py_TARGET_ABI3T`. + If both are defined, use the smaller value. + (If :c:macro:`Py_LIMITED_API` is ``3``; use + :c:expr:`Py_PACK_VERSION(3, 2)` instead of ``3``.) Otherwise, it should be set to :c:macro:`PY_VERSION_HEX`. @@ -352,15 +437,16 @@ The full API is described below for advanced use cases. values of macros such as :c:macro:`Py_LIMITED_API`, :c:macro:`PY_VERSION_HEX` and :c:macro:`Py_GIL_DISABLED`. - .. versionadded:: next + .. versionadded:: 3.15 +.. _limited-c-api: .. _limited-api-list: Contents of Limited API ======================= - -Currently, the :ref:`Limited API ` includes the following items: +This is the definitive list of :ref:`Limited API ` for +Python |version|: .. limited-api-list:: diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 58dd915e04f..aeca4126103 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -33,6 +33,13 @@ under :ref:`reference counting `. The members must not be accessed directly; instead use macros such as :c:macro:`Py_REFCNT` and :c:macro:`Py_TYPE`. + In the :ref:`Stable ABI ` for Free-Threaded Builds (``abi3t``), + this struct is opaque; its size and layout may change between + Python versions. + In Stable ABI for non-free-threaded builds (``abi3``), the + :c:member:`!ob_refcnt` and :c:member:`!ob_type` fields are available, + but using them directly is discouraged. + .. c:member:: Py_ssize_t ob_refcnt The object's reference count, as returned by :c:macro:`Py_REFCNT`. @@ -48,6 +55,19 @@ under :ref:`reference counting `. Do not use this field directly; use :c:macro:`Py_TYPE` and :c:func:`Py_SET_TYPE` instead. + .. c:member:: PyMutex ob_mutex + + A :ref:`per-object lock `, present only in the :term:`free-threaded ` + build (when :c:macro:`Py_GIL_DISABLED` is defined). + + This field is **reserved for use by the critical section API** + (:c:macro:`Py_BEGIN_CRITICAL_SECTION` / :c:macro:`Py_END_CRITICAL_SECTION`). + Do **not** lock it directly with ``PyMutex_Lock``; doing so can cause + deadlocks. If you need your own lock, add a separate :c:type:`PyMutex` + field to your object struct. + + .. versionadded:: 3.13 + .. c:type:: PyVarObject @@ -59,6 +79,19 @@ under :ref:`reference counting `. instead use macros such as :c:macro:`Py_SIZE`, :c:macro:`Py_REFCNT` and :c:macro:`Py_TYPE`. + In the :ref:`Stable ABI ` for Free-Threaded Builds (``abi3t``), + this struct is opaque; its size and layout may change between + Python versions. + In Stable ABI for non-free-threaded builds (``abi3``), the + :c:member:`!ob_base` and :c:member:`!ob_size` fields are available, + but using them directly is discouraged. + + .. c:member:: PyObject ob_base + + Common object header. + Typically, this field is not accessed directly; instead + :c:type:`!PyVarObject` can be cast to :c:type:`PyObject`. + .. c:member:: Py_ssize_t ob_size A size field, whose contents should be considered an object's internal @@ -280,6 +313,8 @@ Implementing functions and methods Name of the method. + A ``NULL`` *ml_name* marks the end of a :c:type:`!PyMethodDef` array. + .. c:member:: PyCFunction ml_meth Pointer to the C implementation. @@ -408,7 +443,7 @@ There are these calling conventions: These two constants are not used to indicate the calling convention but the -binding when use with methods of classes. These may not be used for functions +binding when used with methods of classes. These may not be used for functions defined for modules. At most one of these flags may be set for any given method. @@ -447,6 +482,25 @@ definition with the same method name. slot. This is helpful because calls to PyCFunctions are optimized more than wrapper object calls. + +.. c:var:: PyTypeObject PyCMethod_Type + + The type object corresponding to Python C method objects. This is + available as :class:`types.BuiltinMethodType` in the Python layer. + + +.. c:function:: int PyCMethod_Check(PyObject *op) + + Return true if *op* is an instance of the :c:type:`PyCMethod_Type` type + or a subtype of it. This function always succeeds. + + +.. c:function:: int PyCMethod_CheckExact(PyObject *op) + + This is the same as :c:func:`PyCMethod_Check`, but does not account for + subtypes. + + .. c:function:: PyObject * PyCMethod_New(PyMethodDef *ml, PyObject *self, PyObject *module, PyTypeObject *cls) Turn *ml* into a Python :term:`callable` object. @@ -472,6 +526,24 @@ definition with the same method name. .. versionadded:: 3.9 +.. c:var:: PyTypeObject PyCFunction_Type + + The type object corresponding to Python C function objects. This is + available as :class:`types.BuiltinFunctionType` in the Python layer. + + +.. c:function:: int PyCFunction_Check(PyObject *op) + + Return true if *op* is an instance of the :c:type:`PyCFunction_Type` type + or a subtype of it. This function always succeeds. + + +.. c:function:: int PyCFunction_CheckExact(PyObject *op) + + This is the same as :c:func:`PyCFunction_Check`, but does not account for + subtypes. + + .. c:function:: PyObject * PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module) Equivalent to ``PyCMethod_New(ml, self, module, NULL)``. @@ -482,6 +554,62 @@ definition with the same method name. Equivalent to ``PyCMethod_New(ml, self, NULL, NULL)``. +.. c:function:: int PyCFunction_GetFlags(PyObject *func) + + Get the function's flags on *func* as they were passed to + :c:member:`~PyMethodDef.ml_flags`. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns the function's flags on success, and ``-1`` with an + exception set on failure. + + +.. c:function:: int PyCFunction_GET_FLAGS(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetFlags`, but without error + or type checking. + + +.. c:function:: PyCFunction PyCFunction_GetFunction(PyObject *func) + + Get the function pointer on *func* as it was passed to + :c:member:`~PyMethodDef.ml_meth`. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns the function pointer on success, and ``NULL`` with an + exception set on failure. + + +.. c:function:: int PyCFunction_GET_FUNCTION(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetFunction`, but without error + or type checking. + + +.. c:function:: PyObject *PyCFunction_GetSelf(PyObject *func) + + Get the "self" object on *func*. This is the object that would be passed + to the first argument of a :c:type:`PyCFunction`. For C function objects + created through a :c:type:`PyMethodDef` on a :c:type:`PyModuleDef`, this + is the resulting module object. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns a :term:`borrowed reference` to the "self" object + on success, and ``NULL`` with an exception set on failure. + + +.. c:function:: PyObject *PyCFunction_GET_SELF(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetSelf`, but without error or + type checking. + + Accessing attributes of extension types --------------------------------------- @@ -605,14 +733,12 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: entry indicates an offset from the subclass-specific data, rather than from ``PyObject``. - Can only be used as part of :c:member:`Py_tp_members ` + Can only be used as part of the :c:data:`Py_tp_members` :c:type:`slot ` when creating a class using negative :c:member:`~PyType_Spec.basicsize`. It is mandatory in that case. - - This flag is only used in :c:type:`PyType_Slot`. - When setting :c:member:`~PyTypeObject.tp_members` during - class creation, Python clears it and sets + When setting :c:member:`~PyTypeObject.tp_members` from the slot during + class creation, Python clears the flag and sets :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct. .. index:: diff --git a/Doc/c-api/subinterpreters.rst b/Doc/c-api/subinterpreters.rst new file mode 100644 index 00000000000..83c3fc3d801 --- /dev/null +++ b/Doc/c-api/subinterpreters.rst @@ -0,0 +1,491 @@ +.. highlight:: c + +.. _sub-interpreter-support: + +Multiple interpreters in a Python process +========================================= + +While in most uses, you will only embed a single Python interpreter, there +are cases where you need to create several independent interpreters in the +same process and perhaps even in the same thread. Sub-interpreters allow +you to do that. + +The "main" interpreter is the first one created when the runtime initializes. +It is usually the only Python interpreter in a process. Unlike sub-interpreters, +the main interpreter has unique process-global responsibilities like signal +handling. It is also responsible for execution during runtime initialization and +is usually the active interpreter during runtime finalization. The +:c:func:`PyInterpreterState_Main` function returns a pointer to its state. + +You can switch between sub-interpreters using the :c:func:`PyThreadState_Swap` +function. You can create and destroy them using the following functions: + + +.. c:type:: PyInterpreterConfig + + Structure containing most parameters to configure a sub-interpreter. + Its values are used only in :c:func:`Py_NewInterpreterFromConfig` and + never modified by the runtime. + + .. versionadded:: 3.12 + + Structure fields: + + .. c:member:: int use_main_obmalloc + + If this is ``0`` then the sub-interpreter will use its own + "object" allocator state. + Otherwise it will use (share) the main interpreter's. + + If this is ``0`` then + :c:member:`~PyInterpreterConfig.check_multi_interp_extensions` + must be ``1`` (non-zero). + If this is ``1`` then :c:member:`~PyInterpreterConfig.gil` + must not be :c:macro:`PyInterpreterConfig_OWN_GIL`. + + .. c:member:: int allow_fork + + If this is ``0`` then the runtime will not support forking the + process in any thread where the sub-interpreter is currently active. + Otherwise fork is unrestricted. + + Note that the :mod:`subprocess` module still works + when fork is disallowed. + + .. c:member:: int allow_exec + + If this is ``0`` then the runtime will not support replacing the + current process via exec (e.g. :func:`os.execv`) in any thread + where the sub-interpreter is currently active. + Otherwise exec is unrestricted. + + Note that the :mod:`subprocess` module still works + when exec is disallowed. + + .. c:member:: int allow_threads + + If this is ``0`` then the sub-interpreter's :mod:`threading` module + won't create threads. + Otherwise threads are allowed. + + .. c:member:: int allow_daemon_threads + + If this is ``0`` then the sub-interpreter's :mod:`threading` module + won't create daemon threads. + Otherwise daemon threads are allowed (as long as + :c:member:`~PyInterpreterConfig.allow_threads` is non-zero). + + .. c:member:: int check_multi_interp_extensions + + If this is ``0`` then all extension modules may be imported, + including legacy (single-phase init) modules, + in any thread where the sub-interpreter is currently active. + Otherwise only multi-phase init extension modules + (see :pep:`489`) may be imported. + (Also see :c:macro:`Py_mod_multiple_interpreters`.) + + This must be ``1`` (non-zero) if + :c:member:`~PyInterpreterConfig.use_main_obmalloc` is ``0``. + + .. c:member:: int gil + + This determines the operation of the GIL for the sub-interpreter. + It may be one of the following: + + .. c:namespace:: NULL + + .. c:macro:: PyInterpreterConfig_DEFAULT_GIL + + Use the default selection (:c:macro:`PyInterpreterConfig_SHARED_GIL`). + + .. c:macro:: PyInterpreterConfig_SHARED_GIL + + Use (share) the main interpreter's GIL. + + .. c:macro:: PyInterpreterConfig_OWN_GIL + + Use the sub-interpreter's own GIL. + + If this is :c:macro:`PyInterpreterConfig_OWN_GIL` then + :c:member:`PyInterpreterConfig.use_main_obmalloc` must be ``0``. + + +.. c:function:: PyStatus Py_NewInterpreterFromConfig(PyThreadState **tstate_p, const PyInterpreterConfig *config) + + .. index:: + pair: module; builtins + pair: module; __main__ + pair: module; sys + single: stdout (in module sys) + single: stderr (in module sys) + single: stdin (in module sys) + + Create a new sub-interpreter. This is an (almost) totally separate environment + for the execution of Python code. In particular, the new interpreter has + separate, independent versions of all imported modules, including the + fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. The + table of loaded modules (``sys.modules``) and the module search path + (``sys.path``) are also separate. The new environment has no ``sys.argv`` + variable. It has new standard I/O stream file objects ``sys.stdin``, + ``sys.stdout`` and ``sys.stderr`` (however these refer to the same underlying + file descriptors). + + The given *config* controls the options with which the interpreter + is initialized. + + Upon success, *tstate_p* will be set to the first :term:`thread state` + created in the new sub-interpreter. This thread state is + :term:`attached `. + Note that no actual thread is created; see the discussion of thread states + below. If creation of the new interpreter is unsuccessful, + *tstate_p* is set to ``NULL``; + no exception is set since the exception state is stored in the + :term:`attached thread state`, which might not exist. + + Like all other Python/C API functions, an :term:`attached thread state` + must be present before calling this function, but it might be detached upon + returning. On success, the returned thread state will be :term:`attached `. + If the sub-interpreter is created with its own :term:`GIL` then the + :term:`attached thread state` of the calling interpreter will be detached. + When the function returns, the new interpreter's :term:`thread state` + will be :term:`attached ` to the current thread and + the previous interpreter's :term:`attached thread state` will remain detached. + + .. versionadded:: 3.12 + + Sub-interpreters are most effective when isolated from each other, + with certain functionality restricted:: + + PyInterpreterConfig config = { + .use_main_obmalloc = 0, + .allow_fork = 0, + .allow_exec = 0, + .allow_threads = 1, + .allow_daemon_threads = 0, + .check_multi_interp_extensions = 1, + .gil = PyInterpreterConfig_OWN_GIL, + }; + PyThreadState *tstate = NULL; + PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + + Note that the config is used only briefly and does not get modified. + During initialization the config's values are converted into various + :c:type:`PyInterpreterState` values. A read-only copy of the config + may be stored internally on the :c:type:`PyInterpreterState`. + + .. index:: + single: Py_FinalizeEx (C function) + single: Py_Initialize (C function) + + Extension modules are shared between (sub-)interpreters as follows: + + * For modules using multi-phase initialization, + e.g. :c:func:`PyModule_FromDefAndSpec`, a separate module object is + created and initialized for each interpreter. + Only C-level static and global variables are shared between these + module objects. + + * For modules using legacy + :ref:`single-phase initialization `, + e.g. :c:func:`PyModule_Create`, the first time a particular extension + is imported, it is initialized normally, and a (shallow) copy of its + module's dictionary is squirreled away. + When the same extension is imported by another (sub-)interpreter, a new + module is initialized and filled with the contents of this copy; the + extension's ``init`` function is not called. + Objects in the module's dictionary thus end up shared across + (sub-)interpreters, which might cause unwanted behavior (see + `Bugs and caveats`_ below). + + Note that this is different from what happens when an extension is + imported after the interpreter has been completely re-initialized by + calling :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that + case, the extension's ``initmodule`` function *is* called again. + As with multi-phase initialization, this means that only C-level static + and global variables are shared between these modules. + + .. index:: single: close (in module os) + + +.. c:function:: PyThreadState* Py_NewInterpreter(void) + + .. index:: + pair: module; builtins + pair: module; __main__ + pair: module; sys + single: stdout (in module sys) + single: stderr (in module sys) + single: stdin (in module sys) + + Create a new sub-interpreter. This is essentially just a wrapper + around :c:func:`Py_NewInterpreterFromConfig` with a config that + preserves the existing behavior. The result is an unisolated + sub-interpreter that shares the main interpreter's GIL, allows + fork/exec, allows daemon threads, and allows single-phase init + modules. + + +.. c:function:: void Py_EndInterpreter(PyThreadState *tstate) + + .. index:: single: Py_FinalizeEx (C function) + + Destroy the (sub-)interpreter represented by the given :term:`thread state`. + The given thread state must be :term:`attached `. + When the call returns, there will be no :term:`attached thread state`. + All thread states associated with this interpreter are destroyed. + + :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that + haven't been explicitly destroyed at that point. + + +.. _per-interpreter-gil: + +A per-interpreter GIL +--------------------- + +.. versionadded:: 3.12 + +Using :c:func:`Py_NewInterpreterFromConfig` you can create +a sub-interpreter that is completely isolated from other interpreters, +including having its own GIL. The most important benefit of this +isolation is that such an interpreter can execute Python code without +being blocked by other interpreters or blocking any others. Thus a +single Python process can truly take advantage of multiple CPU cores +when running Python code. The isolation also encourages a different +approach to concurrency than that of just using threads. +(See :pep:`554` and :pep:`684`.) + +Using an isolated interpreter requires vigilance in preserving that +isolation. That especially means not sharing any objects or mutable +state without guarantees about thread-safety. Even objects that are +otherwise immutable (e.g. ``None``, ``(1, 5)``) can't normally be shared +because of the refcount. One simple but less-efficient approach around +this is to use a global lock around all use of some state (or object). +Alternately, effectively immutable objects (like integers or strings) +can be made safe in spite of their refcounts by making them :term:`immortal`. +In fact, this has been done for the builtin singletons, small integers, +and a number of other builtin objects. + +If you preserve isolation then you will have access to proper multi-core +computing without the complications that come with free-threading. +Failure to preserve isolation will expose you to the full consequences +of free-threading, including races and hard-to-debug crashes. + +Aside from that, one of the main challenges of using multiple isolated +interpreters is how to communicate between them safely (not break +isolation) and efficiently. The runtime and stdlib do not provide +any standard approach to this yet. A future stdlib module would help +mitigate the effort of preserving isolation and expose effective tools +for communicating (and sharing) data between interpreters. + + +Bugs and caveats +---------------- + +Because sub-interpreters (and the main interpreter) are part of the same +process, the insulation between them isn't perfect --- for example, using +low-level file operations like :func:`os.close` they can +(accidentally or maliciously) affect each other's open files. Because of the +way extensions are shared between (sub-)interpreters, some extensions may not +work properly; this is especially likely when using single-phase initialization +or (static) global variables. +It is possible to insert objects created in one sub-interpreter into +a namespace of another (sub-)interpreter; this should be avoided if possible. + +Special care should be taken to avoid sharing user-defined functions, +methods, instances or classes between sub-interpreters, since import +operations executed by such objects may affect the wrong (sub-)interpreter's +dictionary of loaded modules. It is equally important to avoid sharing +objects from which the above are reachable. + +Also note that combining this functionality with ``PyGILState_*`` APIs +is delicate, because these APIs assume a bijection between Python thread states +and OS-level threads, an assumption broken by the presence of sub-interpreters. +It is highly recommended that you don't switch sub-interpreters between a pair +of matching :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` calls. +Furthermore, extensions (such as :mod:`ctypes`) using these APIs to allow calling +of Python code from non-Python created threads will probably be broken when using +sub-interpreters. + + +High-level APIs +--------------- + +.. c:type:: PyInterpreterState + + This data structure represents the state shared by a number of cooperating + threads. Threads belonging to the same interpreter share their module + administration and a few other internal items. There are no public members in + this structure. + + Threads belonging to different interpreters initially share nothing, except + process state like available memory, open file descriptors and such. The global + interpreter lock is also shared by all threads, regardless of to which + interpreter they belong. + + .. versionchanged:: 3.12 + + :pep:`684` introduced the possibility + of a :ref:`per-interpreter GIL `. + See :c:func:`Py_NewInterpreterFromConfig`. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Get(void) + + Get the current interpreter. + + Issue a fatal error if there is no :term:`attached thread state`. + It cannot return NULL. + + .. versionadded:: 3.9 + + +.. c:function:: int64_t PyInterpreterState_GetID(PyInterpreterState *interp) + + Return the interpreter's unique ID. If there was any error in doing + so then ``-1`` is returned and an error is set. + + The caller must have an :term:`attached thread state`. + + .. versionadded:: 3.7 + + +.. c:function:: PyObject* PyInterpreterState_GetDict(PyInterpreterState *interp) + + Return a dictionary in which interpreter-specific data may be stored. + If this function returns ``NULL`` then no exception has been raised and + the caller should assume no interpreter-specific dict is available. + + This is not a replacement for :c:func:`PyModule_GetState()`, which + extensions should use to store interpreter-specific state information. + + The returned dictionary is borrowed from the interpreter and is valid until + interpreter shutdown. + + .. versionadded:: 3.8 + + +.. c:type:: PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) + + Type of a frame evaluation function. + + The *throwflag* parameter is used by the ``throw()`` method of generators: + if non-zero, handle the current exception. + + .. versionchanged:: 3.9 + The function now takes a *tstate* parameter. + + .. versionchanged:: 3.11 + The *frame* parameter changed from ``PyFrameObject*`` to ``_PyInterpreterFrame*``. + + +.. c:function:: _PyFrameEvalFunction _PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) + + Get the frame evaluation function. + + See the :pep:`523` "Adding a frame evaluation API to CPython". + + .. versionadded:: 3.9 + + +.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) + + Set the frame evaluation function. + + See the :pep:`523` "Adding a frame evaluation API to CPython". + + .. versionadded:: 3.9 + +.. c:function:: void _PyInterpreterState_SetEvalFrameAllowSpecialization(PyInterpreterState *interp, int allow_specialization) + + Enables or disables specialization why a custom frame evaluator is in place. + + If *allow_specialization* is non-zero, the adaptive specializer will + continue to specialize bytecodes even though a custom eval frame function + is set. When *allow_specialization* is zero, setting a custom eval frame + disables specialization. The standard interpreter loop will continue to deopt + while a frame evaluation API is in place - the frame evaluation function needs + to handle the specialized opcodes to take advantage of this. + + .. versionadded:: 3.15 + +.. c:function:: int _PyInterpreterState_IsSpecializationEnabled(PyInterpreterState *interp) + + Return non-zero if adaptive specialization is enabled for the interpreter. + Specialization is enabled when no custom eval frame function is set, or + when one is set with *allow_specialization* enabled. + + .. versionadded:: 3.15 + + +Low-level APIs +-------------- + +All of the following functions must be called after :c:func:`Py_Initialize`. + +.. versionchanged:: 3.7 + :c:func:`Py_Initialize()` now initializes the :term:`GIL` + and sets an :term:`attached thread state`. + + +.. c:function:: PyInterpreterState* PyInterpreterState_New() + + Create a new interpreter state object. An :term:`attached thread state` is not needed, + but may optionally exist if it is necessary to serialize calls to this + function. + + .. audit-event:: cpython.PyInterpreterState_New "" c.PyInterpreterState_New + + +.. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp) + + Reset all information in an interpreter state object. There must be + an :term:`attached thread state` for the interpreter. + + .. audit-event:: cpython.PyInterpreterState_Clear "" c.PyInterpreterState_Clear + + +.. c:function:: void PyInterpreterState_Delete(PyInterpreterState *interp) + + Destroy an interpreter state object. There **should not** be an + :term:`attached thread state` for the target interpreter. The interpreter + state must have been reset with a previous call to :c:func:`PyInterpreterState_Clear`. + + +.. _advanced-debugging: + +Advanced debugger support +------------------------- + +These functions are only intended to be used by advanced debugging tools. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Head() + + Return the interpreter state object at the head of the list of all such objects. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Main() + + Return the main interpreter state object. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Next(PyInterpreterState *interp) + + Return the next interpreter state object after *interp* from the list of all + such objects. + + +.. c:function:: PyThreadState * PyInterpreterState_ThreadHead(PyInterpreterState *interp) + + Return the pointer to the first :c:type:`PyThreadState` object in the list of + threads associated with the interpreter *interp*. + + +.. c:function:: PyThreadState* PyThreadState_Next(PyThreadState *tstate) + + Return the next thread state object after *tstate* from the list of all such + objects belonging to the same :c:type:`PyInterpreterState` object. diff --git a/Doc/c-api/synchronization.rst b/Doc/c-api/synchronization.rst new file mode 100644 index 00000000000..53c9faeae35 --- /dev/null +++ b/Doc/c-api/synchronization.rst @@ -0,0 +1,308 @@ +.. highlight:: c + +.. _synchronization: + +Synchronization primitives +========================== + +The C-API provides a basic mutual exclusion lock. + +.. c:type:: PyMutex + + A mutual exclusion lock. The :c:type:`!PyMutex` should be initialized to + zero to represent the unlocked state. For example:: + + PyMutex mutex = {0}; + + Instances of :c:type:`!PyMutex` should not be copied or moved. Both the + contents and address of a :c:type:`!PyMutex` are meaningful, and it must + remain at a fixed, writable location in memory. + + .. note:: + + A :c:type:`!PyMutex` currently occupies one byte, but the size should be + considered unstable. The size may change in future Python releases + without a deprecation period. + + .. versionadded:: 3.13 + +.. c:function:: void PyMutex_Lock(PyMutex *m) + + Lock mutex *m*. If another thread has already locked it, the calling + thread will block until the mutex is unlocked. While blocked, the thread + will temporarily detach the :term:`thread state ` if one exists. + + .. versionadded:: 3.13 + +.. c:function:: void PyMutex_Unlock(PyMutex *m) + + Unlock mutex *m*. The mutex must be locked --- otherwise, the function will + issue a fatal error. + + .. versionadded:: 3.13 + +.. c:function:: int PyMutex_IsLocked(PyMutex *m) + + Returns non-zero if the mutex *m* is currently locked, zero otherwise. + + .. note:: + + This function is intended for use in assertions and debugging only and + should not be used to make concurrency control decisions, as the lock + state may change immediately after the check. + + .. versionadded:: 3.14 + +.. _python-critical-section-api: + +Python critical section API +--------------------------- + +The critical section API provides a deadlock avoidance layer on top of +per-object locks for :term:`free-threaded ` CPython. They are +intended to replace reliance on the :term:`global interpreter lock`, and are +no-ops in versions of Python with the global interpreter lock. + +Critical sections are intended to be used for custom types implemented +in C-API extensions. They should generally not be used with built-in types like +:class:`list` and :class:`dict` because their public C-APIs +already use critical sections internally, with the notable +exception of :c:func:`PyDict_Next`, which requires critical section +to be acquired externally. + +Critical sections avoid deadlocks by implicitly suspending active critical +sections, hence, they do not provide exclusive access such as provided by +traditional locks like :c:type:`PyMutex`. When a critical section is started, +the per-object lock for the object is acquired. If the code executed inside the +critical section calls C-API functions then it can suspend the critical section thereby +releasing the per-object lock, so other threads can acquire the per-object lock +for the same object. + +Variants that accept :c:type:`PyMutex` pointers rather than Python objects are also +available. Use these variants to start a critical section in a situation where +there is no :c:type:`PyObject` -- for example, when working with a C type that +does not extend or wrap :c:type:`PyObject` but still needs to call into the C +API in a manner that might lead to deadlocks. + +The functions and structs used by the macros are exposed for cases +where C macros are not available. They should only be used as in the +given macro expansions. Note that the sizes and contents of the structures may +change in future Python versions. + +.. note:: + + Operations that need to lock two objects at once must use + :c:macro:`Py_BEGIN_CRITICAL_SECTION2`. You *cannot* use nested critical + sections to lock more than one object at once, because the inner critical + section may suspend the outer critical sections. This API does not provide + a way to lock more than two objects at once. + +Example usage:: + + static PyObject * + set_field(MyObject *self, PyObject *value) + { + Py_BEGIN_CRITICAL_SECTION(self); + Py_SETREF(self->field, Py_XNewRef(value)); + Py_END_CRITICAL_SECTION(); + Py_RETURN_NONE; + } + +In the above example, :c:macro:`Py_SETREF` calls :c:macro:`Py_DECREF`, which +can call arbitrary code through an object's deallocation function. The critical +section API avoids potential deadlocks due to reentrancy and lock ordering +by allowing the runtime to temporarily suspend the critical section if the +code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION(op) + + Acquires the per-object lock for the object *op* and begins a + critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection _py_cs; + PyCriticalSection_Begin(&_py_cs, (PyObject*)(op)) + + In the default build, this macro expands to ``{``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION_MUTEX(m) + + Locks the mutex *m* and begins a critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection _py_cs; + PyCriticalSection_BeginMutex(&_py_cs, m) + + Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION`, there is no cast for + the argument of the macro - it must be a :c:type:`PyMutex` pointer. + + On the default build, this macro expands to ``{``. + + .. versionadded:: 3.14 + +.. c:macro:: Py_END_CRITICAL_SECTION() + + Ends the critical section and releases the per-object lock. + + In the free-threaded build, this macro expands to:: + + PyCriticalSection_End(&_py_cs); + } + + In the default build, this macro expands to ``}``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION2(a, b) + + Acquires the per-object locks for the objects *a* and *b* and begins a + critical section. The locks are acquired in a consistent order (lowest + address first) to avoid lock ordering deadlocks. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection2 _py_cs2; + PyCriticalSection2_Begin(&_py_cs2, (PyObject*)(a), (PyObject*)(b)) + + In the default build, this macro expands to ``{``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) + + Locks the mutexes *m1* and *m2* and begins a critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection2 _py_cs2; + PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) + + Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION2`, there is no cast for + the arguments of the macro - they must be :c:type:`PyMutex` pointers. + + On the default build, this macro expands to ``{``. + + .. versionadded:: 3.14 + +.. c:macro:: Py_END_CRITICAL_SECTION2() + + Ends the critical section and releases the per-object locks. + + In the free-threaded build, this macro expands to:: + + PyCriticalSection2_End(&_py_cs2); + } + + In the default build, this macro expands to ``}``. + + .. versionadded:: 3.13 + + +Legacy locking APIs +------------------- + +These APIs are obsolete since Python 3.13 with the introduction of +:c:type:`PyMutex`. + +.. versionchanged:: 3.15 + These APIs are now a simple wrapper around ``PyMutex``. + + +.. c:type:: PyThread_type_lock + + A pointer to a mutual exclusion lock. + + +.. c:type:: PyLockStatus + + The result of acquiring a lock with a timeout. + + .. c:namespace:: NULL + + .. c:enumerator:: PY_LOCK_FAILURE + + Failed to acquire the lock. + + .. c:enumerator:: PY_LOCK_ACQUIRED + + The lock was successfully acquired. + + .. c:enumerator:: PY_LOCK_INTR + + The lock was interrupted by a signal. + + +.. c:function:: PyThread_type_lock PyThread_allocate_lock(void) + + Allocate a new lock. + + On success, this function returns a lock; on failure, this + function returns ``0`` without an exception set. + + The caller does not need to hold an :term:`attached thread state`. + + .. versionchanged:: 3.15 + This function now always uses :c:type:`PyMutex`. In prior versions, this + would use a lock provided by the operating system. + + +.. c:function:: void PyThread_free_lock(PyThread_type_lock lock) + + Destroy *lock*. The lock should not be held by any thread when calling + this. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: PyLockStatus PyThread_acquire_lock_timed(PyThread_type_lock lock, long long microseconds, int intr_flag) + + Acquire *lock* with a timeout. + + This will wait for *microseconds* microseconds to acquire the lock. If the + timeout expires, this function returns :c:enumerator:`PY_LOCK_FAILURE`. + If *microseconds* is ``-1``, this will wait indefinitely until the lock has + been released. + + If *intr_flag* is ``1``, acquiring the lock may be interrupted by a signal, + in which case this function returns :c:enumerator:`PY_LOCK_INTR`. Upon + interruption, it's generally expected that the caller makes a call to + :c:func:`Py_MakePendingCalls` to propagate an exception to Python code. + + If the lock is successfully acquired, this function returns + :c:enumerator:`PY_LOCK_ACQUIRED`. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: int PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) + + Acquire *lock*. + + If *waitflag* is ``1`` and another thread currently holds the lock, this + function will wait until the lock can be acquired and will always return + ``1``. + + If *waitflag* is ``0`` and another thread holds the lock, this function will + not wait and instead return ``0``. If the lock is not held by any other + thread, then this function will acquire it and return ``1``. + + Unlike :c:func:`PyThread_acquire_lock_timed`, acquiring the lock cannot be + interrupted by a signal. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: int PyThread_release_lock(PyThread_type_lock lock) + + Release *lock*. If *lock* is not held, then this function issues a + fatal error. + + The caller does not need to hold an :term:`attached thread state`. diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index b34936dd55e..ee73c1c8ada 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -123,6 +123,24 @@ Operating System Utilities This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do not call those functions directly! + +.. c:function:: int PyOS_InterruptOccurred(void) + + Check if a :c:macro:`!SIGINT` signal has been received. + + Returns ``1`` if a :c:macro:`!SIGINT` has occurred and clears the signal flag, + or ``0`` otherwise. + + In most cases, you should prefer :c:func:`PyErr_CheckSignals` over this function. + :c:func:`!PyErr_CheckSignals` invokes the appropriate signal handlers + for all pending signals, allowing Python code to handle the signal properly. + This function only detects :c:macro:`!SIGINT` and does not invoke any Python + signal handlers. + + This function is async-signal-safe and this function cannot fail. + The caller must hold an :term:`attached thread state`. + + .. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size) .. warning:: @@ -268,7 +286,7 @@ accessible to C code. They all work with the current interpreter thread's If the non-existing object should not be treated as a failure, you can use :c:func:`PySys_GetOptionalAttr` instead. - .. versionadded:: next + .. versionadded:: 3.15 .. c:function:: PyObject *PySys_GetAttrString(const char *name) @@ -279,7 +297,7 @@ accessible to C code. They all work with the current interpreter thread's If the non-existing object should not be treated as a failure, you can use :c:func:`PySys_GetOptionalAttrString` instead. - .. versionadded:: next + .. versionadded:: 3.15 .. c:function:: int PySys_GetOptionalAttr(PyObject *name, PyObject **result) @@ -293,7 +311,7 @@ accessible to C code. They all work with the current interpreter thread's * Set an exception, set *\*result* to ``NULL``, and return ``-1``, if an error occurred. - .. versionadded:: next + .. versionadded:: 3.15 .. c:function:: int PySys_GetOptionalAttrString(const char *name, PyObject **result) @@ -301,7 +319,7 @@ accessible to C code. They all work with the current interpreter thread's specified as a :c:expr:`const char*` UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. - .. versionadded:: next + .. versionadded:: 3.15 .. c:function:: PyObject *PySys_GetObject(const char *name) @@ -316,14 +334,6 @@ accessible to C code. They all work with the current interpreter thread's case *name* is deleted from the sys module. Returns ``0`` on success, ``-1`` on error. -.. c:function:: void PySys_ResetWarnOptions() - - Reset :data:`sys.warnoptions` to an empty list. This function may be - called prior to :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. - .. c:function:: void PySys_WriteStdout(const char *format, ...) Write the output string described by *format* to :data:`sys.stdout`. No diff --git a/Doc/c-api/threads.rst b/Doc/c-api/threads.rst new file mode 100644 index 00000000000..3b761d0c657 --- /dev/null +++ b/Doc/c-api/threads.rst @@ -0,0 +1,880 @@ +.. highlight:: c + +.. _threads: + +Thread states and the global interpreter lock +============================================= + +.. index:: + single: global interpreter lock + single: interpreter lock + single: lock, interpreter + +Unless on a :term:`free-threaded build` of :term:`CPython`, +the Python interpreter is generally not thread-safe. In order to support +multi-threaded Python programs, there's a global lock, called the :term:`global +interpreter lock` or :term:`GIL`, that must be held by a thread before +accessing Python objects. Without the lock, even the simplest operations +could cause problems in a multi-threaded program: for example, when +two threads simultaneously increment the reference count of the same object, the +reference count could end up being incremented only once instead of twice. + +As such, only a thread that holds the GIL may operate on Python objects or +invoke Python's C API. + +.. index:: single: setswitchinterval (in module sys) + +In order to emulate concurrency, the interpreter regularly tries to switch +threads between bytecode instructions (see :func:`sys.setswitchinterval`). +This is why locks are also necessary for thread-safety in pure-Python code. + +Additionally, the global interpreter lock is released around blocking I/O +operations, such as reading or writing to a file. From the C API, this is done +by :ref:`detaching the thread state `. + + +.. index:: + single: PyThreadState (C type) + +The Python interpreter keeps some thread-local information inside +a data structure called :c:type:`PyThreadState`, known as a :term:`thread state`. +Each thread has a thread-local pointer to a :c:type:`PyThreadState`; a thread state +referenced by this pointer is considered to be :term:`attached `. + +A thread can only have one :term:`attached thread state` at a time. An attached +thread state is typically analogous with holding the GIL, except on +free-threaded builds. On builds with the GIL enabled, attaching a thread state +will block until the GIL can be acquired. However, even on builds with the GIL +disabled, it is still required to have an attached thread state, as the interpreter +needs to keep track of which threads may access Python objects. + +.. note:: + + Even on the free-threaded build, attaching a thread state may block, as the + GIL can be re-enabled or threads might be temporarily suspended (such as during + a garbage collection). + +Generally, there will always be an attached thread state when using Python's +C API, including during embedding and when implementing methods, so it's uncommon +to need to set up a thread state on your own. Only in some specific cases, such +as in a :c:macro:`Py_BEGIN_ALLOW_THREADS` block or in a fresh thread, will the +thread not have an attached thread state. +If uncertain, check if :c:func:`PyThreadState_GetUnchecked` returns ``NULL``. + +If it turns out that you do need to create a thread state, call :c:func:`PyThreadState_New` +followed by :c:func:`PyThreadState_Swap`, or use the dangerous +:c:func:`PyGILState_Ensure` function. + + +.. _detaching-thread-state: + +Detaching the thread state from extension code +---------------------------------------------- + +Most extension code manipulating the :term:`thread state` has the following simple +structure:: + + Save the thread state in a local variable. + ... Do some blocking I/O operation ... + Restore the thread state from the local variable. + +This is so common that a pair of macros exists to simplify it:: + + Py_BEGIN_ALLOW_THREADS + ... Do some blocking I/O operation ... + Py_END_ALLOW_THREADS + +.. index:: + single: Py_BEGIN_ALLOW_THREADS (C macro) + single: Py_END_ALLOW_THREADS (C macro) + +The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a +hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the +block. + +The block above expands to the following code:: + + PyThreadState *_save; + + _save = PyEval_SaveThread(); + ... Do some blocking I/O operation ... + PyEval_RestoreThread(_save); + +.. index:: + single: PyEval_RestoreThread (C function) + single: PyEval_SaveThread (C function) + +Here is how these functions work: + +The attached thread state implies that the GIL is held for the interpreter. +To detach it, :c:func:`PyEval_SaveThread` is called and the result is stored +in a local variable. + +By detaching the thread state, the GIL is released, which allows other threads +to attach to the interpreter and execute while the current thread performs +blocking I/O. When the I/O operation is complete, the old thread state is +reattached by calling :c:func:`PyEval_RestoreThread`, which will wait until +the GIL can be acquired. + +.. note:: + Performing blocking I/O is the most common use case for detaching + the thread state, but it is also useful to call it over long-running + native code that doesn't need access to Python objects or Python's C API. + For example, the standard :mod:`zlib` and :mod:`hashlib` modules detach the + :term:`thread state ` when compressing or hashing + data. + +On a :term:`free-threaded build`, the :term:`GIL` is usually out of the question, +but **detaching the thread state is still required**, because the interpreter +periodically needs to block all threads to get a consistent view of Python objects +without the risk of race conditions. +For example, CPython currently suspends all threads for a short period of time +while running the garbage collector. + +.. warning:: + + Detaching the thread state can lead to unexpected behavior during interpreter + finalization. See :ref:`cautions-regarding-runtime-finalization` for more + details. + + +APIs +^^^^ + +The following macros are normally used without a trailing semicolon; look for +example usage in the Python source distribution. + +.. note:: + + These macros are still necessary on the :term:`free-threaded build` to prevent + deadlocks. + +.. c:macro:: Py_BEGIN_ALLOW_THREADS + + This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. + Note that it contains an opening brace; it must be matched with a following + :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this + macro. + + +.. c:macro:: Py_END_ALLOW_THREADS + + This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains + a closing brace; it must be matched with an earlier + :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of + this macro. + + +.. c:macro:: Py_BLOCK_THREADS + + This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to + :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. + + +.. c:macro:: Py_UNBLOCK_THREADS + + This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to + :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable + declaration. + + +Non-Python created threads +-------------------------- + +When threads are created using the dedicated Python APIs (such as the +:mod:`threading` module), a thread state is automatically associated with them, +However, when a thread is created from native code (for example, by a +third-party library with its own thread management), it doesn't hold an +attached thread state. + +If you need to call Python code from these threads (often this will be part +of a callback API provided by the aforementioned third-party library), +you must first register these threads with the interpreter by +creating a new thread state and attaching it. + +The most robust way to do this is through :c:func:`PyThreadState_New` followed +by :c:func:`PyThreadState_Swap`. + +.. note:: + ``PyThreadState_New`` requires an argument pointing to the desired + interpreter; such a pointer can be acquired via a call to + :c:func:`PyInterpreterState_Get` from the code where the thread was + created. + +For example:: + + /* The return value of PyInterpreterState_Get() from the + function that created this thread. */ + PyInterpreterState *interp = thread_data->interp; + + /* Create a new thread state for the interpreter. It does not start out + attached. */ + PyThreadState *tstate = PyThreadState_New(interp); + + /* Attach the thread state, which will acquire the GIL. */ + PyThreadState_Swap(tstate); + + /* Perform Python actions here. */ + result = CallSomeFunction(); + /* evaluate result or handle exception */ + + /* Destroy the thread state. No Python API allowed beyond this point. */ + PyThreadState_Clear(tstate); + PyThreadState_DeleteCurrent(); + +.. warning:: + + If the interpreter finalized before ``PyThreadState_Swap`` was called, then + ``interp`` will be a dangling pointer! + +.. _gilstate: + +Legacy API +---------- + +Another common pattern to call Python code from a non-Python thread is to use +:c:func:`PyGILState_Ensure` followed by a call to :c:func:`PyGILState_Release`. + +These functions do not work well when multiple interpreters exist in the Python +process. If no Python interpreter has ever been used in the current thread (which +is common for threads created outside Python), ``PyGILState_Ensure`` will create +and attach a thread state for the "main" interpreter (the first interpreter in +the Python process). + +Additionally, these functions have thread-safety issues during interpreter +finalization. Using ``PyGILState_Ensure`` during finalization will likely +crash the process. + +Usage of these functions look like such:: + + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); + + /* Perform Python actions here. */ + result = CallSomeFunction(); + /* evaluate result or handle exception */ + + /* Release the thread. No Python API allowed beyond this point. */ + PyGILState_Release(gstate); + + +.. _fork-and-threads: + +Cautions about fork() +--------------------- + +Another important thing to note about threads is their behaviour in the face +of the C :c:func:`fork` call. On most systems with :c:func:`fork`, after a +process forks only the thread that issued the fork will exist. This has a +concrete impact both on how locks must be handled and on all stored state +in CPython's runtime. + +The fact that only the "current" thread remains +means any locks held by other threads will never be released. Python solves +this for :func:`os.fork` by acquiring the locks it uses internally before +the fork, and releasing them afterwards. In addition, it resets any +:ref:`lock-objects` in the child. When extending or embedding Python, there +is no way to inform Python of additional (non-Python) locks that need to be +acquired before or reset after a fork. OS facilities such as +:c:func:`!pthread_atfork` would need to be used to accomplish the same thing. +Additionally, when extending or embedding Python, calling :c:func:`fork` +directly rather than through :func:`os.fork` (and returning to or calling +into Python) may result in a deadlock by one of Python's internal locks +being held by a thread that is defunct after the fork. +:c:func:`PyOS_AfterFork_Child` tries to reset the necessary locks, but is not +always able to. + +The fact that all other threads go away also means that CPython's +runtime state there must be cleaned up properly, which :func:`os.fork` +does. This means finalizing all other :c:type:`PyThreadState` objects +belonging to the current interpreter and all other +:c:type:`PyInterpreterState` objects. Due to this and the special +nature of the :ref:`"main" interpreter `, +:c:func:`fork` should only be called in that interpreter's "main" +thread, where the CPython global runtime was originally initialized. +The only exception is if :c:func:`exec` will be called immediately +after. + + +High-level APIs +--------------- + +These are the most commonly used types and functions when writing multi-threaded +C extensions. + + +.. c:type:: PyThreadState + + This data structure represents the state of a single thread. The only public + data member is: + + .. c:member:: PyInterpreterState *interp + + This thread's interpreter state. + + +.. c:function:: void PyEval_InitThreads() + + .. index:: + single: PyEval_AcquireThread() + single: PyEval_ReleaseThread() + single: PyEval_SaveThread() + single: PyEval_RestoreThread() + + Deprecated function which does nothing. + + In Python 3.6 and older, this function created the GIL if it didn't exist. + + .. versionchanged:: 3.9 + The function now does nothing. + + .. versionchanged:: 3.7 + This function is now called by :c:func:`Py_Initialize()`, so you don't + have to call it yourself anymore. + + .. versionchanged:: 3.2 + This function cannot be called before :c:func:`Py_Initialize()` anymore. + + .. deprecated:: 3.9 + + .. index:: pair: module; _thread + + +.. c:function:: PyThreadState* PyEval_SaveThread() + + Detach the :term:`attached thread state` and return it. + The thread will have no :term:`thread state` upon returning. + + +.. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) + + Set the :term:`attached thread state` to *tstate*. + The passed :term:`thread state` **should not** be :term:`attached `, + otherwise deadlock ensues. *tstate* will be attached upon returning. + + .. note:: + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + +.. c:function:: PyThreadState* PyThreadState_Get() + + Return the :term:`attached thread state`. If the thread has no attached + thread state, (such as when inside of :c:macro:`Py_BEGIN_ALLOW_THREADS` + block), then this issues a fatal error (so that the caller needn't check + for ``NULL``). + + See also :c:func:`PyThreadState_GetUnchecked`. + +.. c:function:: PyThreadState* PyThreadState_GetUnchecked() + + Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a + fatal error if it is NULL. The caller is responsible to check if the result + is NULL. + + .. versionadded:: 3.13 + In Python 3.5 to 3.12, the function was private and known as + ``_PyThreadState_UncheckedGet()``. + + +.. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) + + Set the :term:`attached thread state` to *tstate*, and return the + :term:`thread state` that was attached prior to calling. + + This function is safe to call without an :term:`attached thread state`; it + will simply return ``NULL`` indicating that there was no prior thread state. + + .. seealso:: + :c:func:`PyEval_ReleaseThread` + + .. note:: + Similar to :c:func:`PyGILState_Ensure`, this function will hang the + thread if the runtime is finalizing. + + +GIL-state APIs +-------------- + +The following functions use thread-local storage, and are not compatible +with sub-interpreters: + +.. c:type:: PyGILState_STATE + + The type of the value returned by :c:func:`PyGILState_Ensure` and passed to + :c:func:`PyGILState_Release`. + + .. c:enumerator:: PyGILState_LOCKED + + The GIL was already held when :c:func:`PyGILState_Ensure` was called. + + .. c:enumerator:: PyGILState_UNLOCKED + + The GIL was not held when :c:func:`PyGILState_Ensure` was called. + +.. c:function:: PyGILState_STATE PyGILState_Ensure() + + Ensure that the current thread is ready to call the Python C API regardless + of the current state of Python, or of the :term:`attached thread state`. This may + be called as many times as desired by a thread as long as each call is + matched with a call to :c:func:`PyGILState_Release`. In general, other + thread-related APIs may be used between :c:func:`PyGILState_Ensure` and + :c:func:`PyGILState_Release` calls as long as the thread state is restored to + its previous state before the Release(). For example, normal usage of the + :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is + acceptable. + + The return value is an opaque "handle" to the :term:`attached thread state` when + :c:func:`PyGILState_Ensure` was called, and must be passed to + :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even + though recursive calls are allowed, these handles *cannot* be shared - each + unique call to :c:func:`PyGILState_Ensure` must save the handle for its call + to :c:func:`PyGILState_Release`. + + When the function returns, there will be an :term:`attached thread state` + and the thread will be able to call arbitrary Python code. Failure is a fatal error. + + .. warning:: + Calling this function when the runtime is finalizing is unsafe. Doing + so will either hang the thread until the program ends, or fully crash + the interpreter in rare cases. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + +.. c:function:: void PyGILState_Release(PyGILState_STATE) + + Release any resources previously acquired. After this call, Python's state will + be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call + (but generally this state will be unknown to the caller, hence the use of the + GILState API). + + Every call to :c:func:`PyGILState_Ensure` must be matched by a call to + :c:func:`PyGILState_Release` on the same thread. + +.. c:function:: PyThreadState* PyGILState_GetThisThreadState() + + Get the :term:`attached thread state` for this thread. May return ``NULL`` if no + GILState API has been used on the current thread. Note that the main thread + always has such a thread-state, even if no auto-thread-state call has been + made on the main thread. This is mainly a helper/diagnostic function. + + .. note:: + This function may return non-``NULL`` even when the :term:`thread state` + is detached. + Prefer :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` + for most cases. + + .. seealso:: :c:func:`PyThreadState_Get` + +.. c:function:: int PyGILState_Check() + + Return ``1`` if the current thread is holding the :term:`GIL` and ``0`` otherwise. + This function can be called from any thread at any time. + Only if it has had its :term:`thread state ` initialized + via :c:func:`PyGILState_Ensure` will it return ``1``. + This is mainly a helper/diagnostic function. It can be useful + for example in callback contexts or memory allocation functions when + knowing that the :term:`GIL` is locked can allow the caller to perform sensitive + actions or otherwise behave differently. + + .. note:: + If the current Python process has ever created a subinterpreter, this + function will *always* return ``1``. Prefer :c:func:`PyThreadState_GetUnchecked` + for most cases. + + .. versionadded:: 3.4 + + +Low-level APIs +-------------- + +.. c:function:: PyThreadState* PyThreadState_New(PyInterpreterState *interp) + + Create a new thread state object belonging to the given interpreter object. + An :term:`attached thread state` is not needed. + +.. c:function:: void PyThreadState_Clear(PyThreadState *tstate) + + Reset all information in a :term:`thread state` object. *tstate* + must be :term:`attached ` + + .. versionchanged:: 3.9 + This function now calls the :c:member:`!PyThreadState.on_delete` callback. + Previously, that happened in :c:func:`PyThreadState_Delete`. + + .. versionchanged:: 3.13 + The :c:member:`!PyThreadState.on_delete` callback was removed. + + +.. c:function:: void PyThreadState_Delete(PyThreadState *tstate) + + Destroy a :term:`thread state` object. *tstate* should not + be :term:`attached ` to any thread. + *tstate* must have been reset with a previous call to + :c:func:`PyThreadState_Clear`. + + +.. c:function:: void PyThreadState_DeleteCurrent(void) + + Detach the :term:`attached thread state` (which must have been reset + with a previous call to :c:func:`PyThreadState_Clear`) and then destroy it. + + No :term:`thread state` will be :term:`attached ` upon + returning. + +.. c:function:: PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) + + Get the current frame of the Python thread state *tstate*. + + Return a :term:`strong reference`. Return ``NULL`` if no frame is currently + executing. + + See also :c:func:`PyEval_GetFrame`. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: uint64_t PyThreadState_GetID(PyThreadState *tstate) + + Get the unique :term:`thread state` identifier of the Python thread state *tstate*. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: PyInterpreterState* PyThreadState_GetInterpreter(PyThreadState *tstate) + + Get the interpreter of the Python thread state *tstate*. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: void PyThreadState_EnterTracing(PyThreadState *tstate) + + Suspend tracing and profiling in the Python thread state *tstate*. + + Resume them using the :c:func:`PyThreadState_LeaveTracing` function. + + .. versionadded:: 3.11 + + +.. c:function:: void PyThreadState_LeaveTracing(PyThreadState *tstate) + + Resume tracing and profiling in the Python thread state *tstate* suspended + by the :c:func:`PyThreadState_EnterTracing` function. + + See also :c:func:`PyEval_SetTrace` and :c:func:`PyEval_SetProfile` + functions. + + .. versionadded:: 3.11 + + +.. c:function:: int PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, void *stack_start_addr, size_t stack_size) + + Set the stack protection start address and stack protection size + of a Python thread state. + + On success, return ``0``. + On failure, set an exception and return ``-1``. + + CPython implements :ref:`recursion control ` for C code by raising + :py:exc:`RecursionError` when it notices that the machine execution stack is close + to overflow. See for example the :c:func:`Py_EnterRecursiveCall` function. + For this, it needs to know the location of the current thread's stack, which it + normally gets from the operating system. + When the stack is changed, for example using context switching techniques like the + Boost library's ``boost::context``, you must call + :c:func:`~PyUnstable_ThreadState_SetStackProtection` to inform CPython of the change. + + Call :c:func:`~PyUnstable_ThreadState_SetStackProtection` either before + or after changing the stack. + Do not call any other Python C API between the call and the stack + change. + + See :c:func:`PyUnstable_ThreadState_ResetStackProtection` for undoing this operation. + + .. versionadded:: 3.15 + + +.. c:function:: void PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) + + Reset the stack protection start address and stack protection size + of a Python thread state to the operating system defaults. + + See :c:func:`PyUnstable_ThreadState_SetStackProtection` for an explanation. + + .. versionadded:: 3.15 + + +.. c:function:: PyObject* PyThreadState_GetDict() + + Return a dictionary in which extensions can store thread-specific state + information. Each extension should use a unique key to use to store state in + the dictionary. It is okay to call this function when no :term:`thread state` + is :term:`attached `. If this function returns + ``NULL``, no exception has been raised and the caller should assume no + thread state is attached. + + +.. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) + + :term:`Attach ` *tstate* to the current thread, + which must not be ``NULL`` or already :term:`attached `. + + The calling thread must not already have an :term:`attached thread state`. + + .. note:: + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.8 + Updated to be consistent with :c:func:`PyEval_RestoreThread`, + :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, + and terminate the current thread if called while the interpreter is finalizing. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + + :c:func:`PyEval_RestoreThread` is a higher-level function which is always + available (even when threads have not been initialized). + + +.. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) + + Detach the :term:`attached thread state`. + The *tstate* argument, which must not be ``NULL``, is only used to check + that it represents the :term:`attached thread state` --- if it isn't, a fatal error is + reported. + + :c:func:`PyEval_SaveThread` is a higher-level function which is always + available (even when threads have not been initialized). + + +Asynchronous notifications +========================== + +A mechanism is provided to make asynchronous notifications to the main +interpreter thread. These notifications take the form of a function +pointer and a void pointer argument. + + +.. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) + + Schedule a function to be called from the main interpreter thread. On + success, ``0`` is returned and *func* is queued for being called in the + main thread. On failure, ``-1`` is returned without setting any exception. + + When successfully queued, *func* will be *eventually* called from the + main interpreter thread with the argument *arg*. It will be called + asynchronously with respect to normally running Python code, but with + both these conditions met: + + * on a :term:`bytecode` boundary; + * with the main thread holding an :term:`attached thread state` + (*func* can therefore use the full C API). + + *func* must return ``0`` on success, or ``-1`` on failure with an exception + set. *func* won't be interrupted to perform another asynchronous + notification recursively, but it can still be interrupted to switch + threads if the :term:`thread state ` is detached. + + This function doesn't need an :term:`attached thread state`. However, to call this + function in a subinterpreter, the caller must have an :term:`attached thread state`. + Otherwise, the function *func* can be scheduled to be called from the wrong interpreter. + + .. warning:: + This is a low-level function, only useful for very special cases. + There is no guarantee that *func* will be called as quick as + possible. If the main thread is busy executing a system call, + *func* won't be called before the system call returns. This + function is generally **not** suitable for calling Python code from + arbitrary C threads. Instead, use the :ref:`PyGILState API`. + + .. versionadded:: 3.1 + + .. versionchanged:: 3.9 + If this function is called in a subinterpreter, the function *func* is + now scheduled to be called from the subinterpreter, rather than being + called from the main interpreter. Each subinterpreter now has its own + list of scheduled calls. + + .. versionchanged:: 3.12 + This function now always schedules *func* to be run in the main + interpreter. + + +.. c:function:: int Py_MakePendingCalls(void) + + Execute all pending calls. This is usually executed automatically by the + interpreter. + + This function returns ``0`` on success, and returns ``-1`` with an exception + set on failure. + + If this is not called in the main thread of the main + interpreter, this function does nothing and returns ``0``. + The caller must hold an :term:`attached thread state`. + + .. versionadded:: 3.1 + + .. versionchanged:: 3.12 + This function only runs pending calls in the main interpreter. + + +.. c:function:: int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) + + Schedule an exception to be raised asynchronously in a thread. + If the thread has a previously scheduled exception, it is overwritten. + + The *id* argument is the thread id of the target thread, as returned by + :c:func:`PyThread_get_thread_ident`. + *exc* is the class of the exception to be raised, or ``NULL`` to clear + the pending exception (if any). + + Return the number of affected thread states. + This is normally ``1`` if *id* is found, even when no change was + made (the given *exc* was already pending, or *exc* is ``NULL`` but + no exception is pending). + If the thread id isn't found, return ``0``. This raises no exceptions. + + To prevent naive misuse, you must write your own C extension to call this. + This function must be called with an :term:`attached thread state`. + This function does not steal any references to *exc*. + This function does not necessarily interrupt system calls such as + :py:func:`~time.sleep`. + + .. versionchanged:: 3.7 + The type of the *id* parameter changed from :c:expr:`long` to + :c:expr:`unsigned long`. + + +Operating system thread APIs +============================ + +.. c:macro:: PYTHREAD_INVALID_THREAD_ID + + Sentinel value for an invalid thread ID. + + This is currently equivalent to ``(unsigned long)-1``. + + +.. c:function:: unsigned long PyThread_start_new_thread(void (*func)(void *), void *arg) + + Start function *func* in a new thread with argument *arg*. + The resulting thread is not intended to be joined. + + *func* must not be ``NULL``, but *arg* may be ``NULL``. + + On success, this function returns the identifier of the new thread; on failure, + this returns :c:macro:`PYTHREAD_INVALID_THREAD_ID`. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: unsigned long PyThread_get_thread_ident(void) + + Return the identifier of the current thread, which will never be zero. + + This function cannot fail, and the caller does not need to hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`threading.get_ident` and :py:attr:`threading.Thread.ident` + expose this identifier to Python. + + +.. c:function:: PyObject *PyThread_GetInfo(void) + + Get general information about the current thread in the form of a + :ref:`struct sequence ` object. This information is + accessible as :py:attr:`sys.thread_info` in Python. + + On success, this returns a new :term:`strong reference` to the thread + information; on failure, this returns ``NULL`` with an exception set. + + The caller must hold an :term:`attached thread state`. + + +.. c:macro:: PY_HAVE_THREAD_NATIVE_ID + + This macro is defined when the system supports native thread IDs. + + +.. c:function:: unsigned long PyThread_get_thread_native_id(void) + + Get the native identifier of the current thread as it was assigned by the operating + system's kernel, which will never be less than zero. + + This function is only available when :c:macro:`PY_HAVE_THREAD_NATIVE_ID` is + defined. + + This function cannot fail, and the caller does not need to hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`threading.get_native_id` + + +.. c:function:: void PyThread_exit_thread(void) + + Terminate the current thread. This function is generally considered unsafe + and should be avoided. It is kept solely for backwards compatibility. + + This function is only safe to call if all functions in the full call + stack are written to safely allow it. + + .. warning:: + + If the current system uses POSIX threads (also known as "pthreads"), + this calls :manpage:`pthread_exit(3)`, which attempts to unwind the stack + and call C++ destructors on some libc implementations. However, if a + ``noexcept`` function is reached, it may terminate the process. + Other systems, such as macOS, do unwinding. + + On Windows, this function calls ``_endthreadex()``, which kills the thread + without calling C++ destructors. + + In any case, there is a risk of corruption on the thread's stack. + + .. deprecated:: 3.14 + + +.. c:function:: void PyThread_init_thread(void) + + Initialize ``PyThread*`` APIs. Python executes this function automatically, + so there's little need to call it from an extension module. + + +.. c:function:: int PyThread_set_stacksize(size_t size) + + Set the stack size of the current thread to *size* bytes. + + This function returns ``0`` on success, ``-1`` if *size* is invalid, or + ``-2`` if the system does not support changing the stack size. This function + does not set exceptions. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: size_t PyThread_get_stacksize(void) + + Return the stack size of the current thread in bytes, or ``0`` if the system's + default stack size is in use. + + The caller does not need to hold an :term:`attached thread state`. diff --git a/Doc/c-api/tls.rst b/Doc/c-api/tls.rst new file mode 100644 index 00000000000..93ac5557141 --- /dev/null +++ b/Doc/c-api/tls.rst @@ -0,0 +1,155 @@ +.. highlight:: c + +.. _thread-local-storage: + +Thread-local storage support +============================ + +The Python interpreter provides low-level support for thread-local storage +(TLS) which wraps the underlying native TLS implementation to support the +Python-level thread-local storage API (:class:`threading.local`). The +CPython C level APIs are similar to those offered by pthreads and Windows: +use a thread key and functions to associate a :c:expr:`void*` value per +thread. + +A :term:`thread state` does *not* need to be :term:`attached ` +when calling these functions; they supply their own locking. + +Note that :file:`Python.h` does not include the declaration of the TLS APIs, +you need to include :file:`pythread.h` to use thread-local storage. + +.. note:: + None of these API functions handle memory management on behalf of the + :c:expr:`void*` values. You need to allocate and deallocate them yourself. + If the :c:expr:`void*` values happen to be :c:expr:`PyObject*`, these + functions don't do refcount operations on them either. + +.. _thread-specific-storage-api: + +Thread-specific storage API +--------------------------- + +The thread-specific storage (TSS) API was introduced to supersede the use of the existing TLS API within the +CPython interpreter. This API uses a new type :c:type:`Py_tss_t` instead of +:c:expr:`int` to represent thread keys. + +.. versionadded:: 3.7 + +.. seealso:: "A New C-API for Thread-Local Storage in CPython" (:pep:`539`) + + +.. c:type:: Py_tss_t + + This data structure represents the state of a thread key, the definition of + which may depend on the underlying TLS implementation, and it has an + internal field representing the key's initialization state. There are no + public members in this structure. + + When :ref:`Py_LIMITED_API ` is not defined, static allocation of + this type by :c:macro:`Py_tss_NEEDS_INIT` is allowed. + + +.. c:macro:: Py_tss_NEEDS_INIT + + This macro expands to the initializer for :c:type:`Py_tss_t` variables. + Note that this macro won't be defined with :ref:`Py_LIMITED_API `. + + +Dynamic allocation +------------------ + +Dynamic allocation of the :c:type:`Py_tss_t`, required in extension modules +built with :ref:`Py_LIMITED_API `, where static allocation of this type +is not possible due to its implementation being opaque at build time. + + +.. c:function:: Py_tss_t* PyThread_tss_alloc() + + Return a value which is the same state as a value initialized with + :c:macro:`Py_tss_NEEDS_INIT`, or ``NULL`` in the case of dynamic allocation + failure. + + +.. c:function:: void PyThread_tss_free(Py_tss_t *key) + + Free the given *key* allocated by :c:func:`PyThread_tss_alloc`, after + first calling :c:func:`PyThread_tss_delete` to ensure any associated + thread locals have been unassigned. This is a no-op if the *key* + argument is ``NULL``. + + .. note:: + A freed key becomes a dangling pointer. You should reset the key to + ``NULL``. + + +Methods +------- + +The parameter *key* of these functions must not be ``NULL``. Moreover, the +behaviors of :c:func:`PyThread_tss_set` and :c:func:`PyThread_tss_get` are +undefined if the given :c:type:`Py_tss_t` has not been initialized by +:c:func:`PyThread_tss_create`. + + +.. c:function:: int PyThread_tss_is_created(Py_tss_t *key) + + Return a non-zero value if the given :c:type:`Py_tss_t` has been initialized + by :c:func:`PyThread_tss_create`. + + +.. c:function:: int PyThread_tss_create(Py_tss_t *key) + + Return a zero value on successful initialization of a TSS key. The behavior + is undefined if the value pointed to by the *key* argument is not + initialized by :c:macro:`Py_tss_NEEDS_INIT`. This function can be called + repeatedly on the same key -- calling it on an already initialized key is a + no-op and immediately returns success. + + +.. c:function:: void PyThread_tss_delete(Py_tss_t *key) + + Destroy a TSS key to forget the values associated with the key across all + threads, and change the key's initialization state to uninitialized. A + destroyed key is able to be initialized again by + :c:func:`PyThread_tss_create`. This function can be called repeatedly on + the same key -- calling it on an already destroyed key is a no-op. + + +.. c:function:: int PyThread_tss_set(Py_tss_t *key, void *value) + + Return a zero value to indicate successfully associating a :c:expr:`void*` + value with a TSS key in the current thread. Each thread has a distinct + mapping of the key to a :c:expr:`void*` value. + + +.. c:function:: void* PyThread_tss_get(Py_tss_t *key) + + Return the :c:expr:`void*` value associated with a TSS key in the current + thread. This returns ``NULL`` if no value is associated with the key in the + current thread. + + +.. _thread-local-storage-api: + +Legacy APIs +----------- + +.. deprecated:: 3.7 + This API is superseded by the + :ref:`thread-specific storage (TSS) API `. + +.. note:: + This version of the API does not support platforms where the native TLS key + is defined in a way that cannot be safely cast to ``int``. On such platforms, + :c:func:`PyThread_create_key` will return immediately with a failure status, + and the other TLS functions will all be no-ops on such platforms. + +Due to the compatibility problem noted above, this version of the API should not +be used in new code. + +.. c:function:: int PyThread_create_key() +.. c:function:: void PyThread_delete_key(int key) +.. c:function:: int PyThread_set_key_value(int key, void *value) +.. c:function:: void* PyThread_get_key_value(int key) +.. c:function:: void PyThread_delete_key_value(int key) +.. c:function:: void PyThread_ReInitTLS() diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 815afddad19..ba4c6b93de4 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -37,6 +37,19 @@ Tuple Objects or ``NULL`` with an exception set on failure. +.. c:function:: PyObject* PyTuple_FromArray(PyObject *const *array, Py_ssize_t size) + + Create a tuple of *size* items and copy references from *array* to the new + tuple. + + *array* can be NULL if *size* is ``0``. + + On success, return a new reference. + On error, set an exception and return ``NULL``. + + .. versionadded:: 3.15 + + .. c:function:: PyObject* PyTuple_Pack(Py_ssize_t n, ...) Return a new tuple object of size *n*, @@ -48,7 +61,7 @@ Tuple Objects .. c:function:: Py_ssize_t PyTuple_Size(PyObject *p) Take a pointer to a tuple object, and return the size of that tuple. - On error, return ``-1`` and with an exception set. + On error, return ``-1`` with an exception set. .. c:function:: Py_ssize_t PyTuple_GET_SIZE(PyObject *p) @@ -86,7 +99,8 @@ Tuple Objects Insert a reference to object *o* at position *pos* of the tuple pointed to by *p*. Return ``0`` on success. If *pos* is out of bounds, return ``-1`` - and set an :exc:`IndexError` exception. + and set an :exc:`IndexError` exception. This function should only be used to fill in brand new tuples; + using it on an existing tuple is thread-unsafe. .. note:: @@ -97,7 +111,7 @@ Tuple Objects .. c:function:: void PyTuple_SET_ITEM(PyObject *p, Py_ssize_t pos, PyObject *o) Like :c:func:`PyTuple_SetItem`, but does no error checking, and should *only* be - used to fill in brand new tuples. + used to fill in brand new tuples, using it on an existing tuple is thread-unsafe. Bounds checking is performed as an assertion if Python is built in :ref:`debug mode ` or :option:`with assertions <--with-assertions>`. @@ -135,8 +149,11 @@ Tuple Objects Struct Sequence Objects ----------------------- -Struct sequence objects are the C equivalent of :func:`~collections.namedtuple` -objects, i.e. a sequence whose items can also be accessed through attributes. +A struct sequence object is a :term:`named tuple`, that is, a sequence +whose items can also be accessed through attributes. +It is similar to :func:`collections.namedtuple`, but provides a slightly +different interface. + To create a struct sequence, you first have to create a specific struct sequence type. @@ -220,6 +237,8 @@ type. .. c:function:: PyObject* PyStructSequence_GetItem(PyObject *p, Py_ssize_t pos) Return the object at position *pos* in the struct sequence pointed to by *p*. + The returned reference is borrowed from the struct sequence *p* + (that is: it is only valid as long as you hold a reference to *p*). Bounds checking is performed as an assertion if Python is built in :ref:`debug mode ` or :option:`with assertions <--with-assertions>`. diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 5bdbff4e0ad..c9bb5c3f09a 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -116,6 +116,20 @@ Type Objects .. versionadded:: 3.12 +.. c:function:: int PyType_Unwatch(int watcher_id, PyObject *type) + + Mark *type* as not watched. This undoes a previous call to + :c:func:`PyType_Watch`. *type* must not be ``NULL``. + + An extension should never call this function with a *watcher_id* that was + not returned to it by a previous call to :c:func:`PyType_AddWatcher`. + + On success, this function returns ``0``. On failure, this function returns + ``-1`` with an exception set. + + .. versionadded:: 3.12 + + .. c:type:: int (*PyType_WatchCallback)(PyObject *type) Type of a type-watcher callback function. @@ -133,6 +147,18 @@ Type Objects Type features are denoted by single bit flags. +.. c:function:: int PyType_FastSubclass(PyTypeObject *type, int flag) + + Return non-zero if the type object *type* sets the subclass flag *flag*. + Subclass flags are denoted by + :c:macro:`Py_TPFLAGS_*_SUBCLASS `. + This function is used by many ``_Check`` functions for common types. + + .. seealso:: + :c:func:`PyObject_TypeCheck`, which is used as a slower alternative in + ``_Check`` functions for types that don't come with subclass flags. + + .. c:function:: int PyType_IS_GC(PyTypeObject *o) Return true if the type object includes support for the cycle detector; this @@ -169,12 +195,14 @@ Type Objects before initialization) and should be paired with :c:func:`PyObject_Free` in :c:member:`~PyTypeObject.tp_free`. + .. c:function:: PyObject* PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) Generic handler for the :c:member:`~PyTypeObject.tp_new` slot of a type object. Creates a new instance using the type's :c:member:`~PyTypeObject.tp_alloc` slot and returns the resulting object. + .. c:function:: int PyType_Ready(PyTypeObject *type) Finalize a type object. This should be called on all type objects to finish @@ -191,6 +219,7 @@ Type Objects GC protocol itself by at least implementing the :c:member:`~PyTypeObject.tp_traverse` handle. + .. c:function:: PyObject* PyType_GetName(PyTypeObject *type) Return the type's name. Equivalent to getting the type's @@ -198,6 +227,7 @@ Type Objects .. versionadded:: 3.11 + .. c:function:: PyObject* PyType_GetQualName(PyTypeObject *type) Return the type's qualified name. Equivalent to getting the @@ -213,6 +243,7 @@ Type Objects .. versionadded:: 3.13 + .. c:function:: PyObject* PyType_GetModuleName(PyTypeObject *type) Return the type's module name. Equivalent to getting the @@ -220,6 +251,7 @@ Type Objects .. versionadded:: 3.13 + .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the @@ -236,11 +268,16 @@ Type Objects :c:func:`PyType_GetSlot` can now accept all types. Previously, it was limited to :ref:`heap types `. + .. c:function:: PyObject* PyType_GetModule(PyTypeObject *type) Return the module object associated with the given type when the type was created using :c:func:`PyType_FromModuleAndSpec`. + The returned reference is :term:`borrowed ` from *type*, + and will be valid as long as you hold a reference to *type*. + Do not release it with :c:func:`Py_DECREF` or similar. + If no module is associated with the given type, sets :py:class:`TypeError` and returns ``NULL``. @@ -250,11 +287,12 @@ Type Objects ``Py_TYPE(self)`` may be a *subclass* of the intended class, and subclasses are not necessarily defined in the same module as their superclass. See :c:type:`PyCMethod` to get the class that defines the method. - See :c:func:`PyType_GetModuleByDef` for cases when :c:type:`!PyCMethod` cannot - be used. + See :c:func:`PyType_GetModuleByToken` for cases when :c:type:`!PyCMethod` + cannot be used. .. versionadded:: 3.9 + .. c:function:: void* PyType_GetModuleState(PyTypeObject *type) Return the state of the module object associated with the given type. @@ -269,10 +307,11 @@ Type Objects .. versionadded:: 3.9 -.. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) - Find the first superclass whose module was created from - the given :c:type:`PyModuleDef` *def*, and return that module. +.. c:function:: PyObject* PyType_GetModuleByToken(PyTypeObject *type, const void *mod_token) + + Find the first superclass whose module has the given + :ref:`module token `, and return that module. If no module is found, raises a :py:class:`TypeError` and returns ``NULL``. @@ -282,16 +321,34 @@ Type Objects and other places where a method's defining class cannot be passed using the :c:type:`PyCMethod` calling convention. + .. versionadded:: 3.15 + + +.. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) + + Find the first superclass whose module was created from the given + :c:type:`PyModuleDef` *def*, or whose :ref:`module token ` + is equal to *def*, and return that module. + + Note that modules created from a :c:type:`PyModuleDef` always have their + token set to the :c:type:`PyModuleDef`'s address. + In other words, this function is equivalent to + :c:func:`PyType_GetModuleByToken`, except that it: + + - returns a borrowed reference, and + - has a non-``void*`` argument type (which is a cosmetic difference in C). + The returned reference is :term:`borrowed ` from *type*, and will be valid as long as you hold a reference to *type*. Do not release it with :c:func:`Py_DECREF` or similar. .. versionadded:: 3.11 -.. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) + +.. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *tp_token, PyTypeObject **result) Find the first superclass in *type*'s :term:`method resolution order` whose - :c:macro:`Py_tp_token` token is equal to the given one. + :c:macro:`Py_tp_token` token is equal to *tp_token*. * If found, set *\*result* to a new :term:`strong reference` to it and return ``1``. @@ -302,10 +359,11 @@ Type Objects The *result* argument may be ``NULL``, in which case *\*result* is not set. Use this if you need only the return value. - The *token* argument may not be ``NULL``. + The *tp_token* argument may not be ``NULL``. .. versionadded:: 3.14 + .. c:function:: int PyUnstable_Type_AssignVersionTag(PyTypeObject *type) Attempt to assign a version tag to the given type. @@ -316,6 +374,16 @@ Type Objects .. versionadded:: 3.12 +.. c:function:: int PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) + + Return true if instances of *type* support creating weak references, false + otherwise. This function always succeeds. *type* must not be ``NULL``. + + .. seealso:: + * :ref:`weakrefobjects` + * :py:mod:`weakref` + + Creating Heap-Allocated Types ............................. @@ -336,8 +404,8 @@ The following functions and structs are used to create The *bases* argument can be used to specify base classes; it can either be only one class or a tuple of classes. - If *bases* is ``NULL``, the *Py_tp_bases* slot is used instead. - If that also is ``NULL``, the *Py_tp_base* slot is used instead. + If *bases* is ``NULL``, the :c:data:`Py_tp_bases` slot is used instead. + If that also is ``NULL``, the :c:data:`Py_tp_base` slot is used instead. If that also is ``NULL``, the new type derives from :class:`object`. The *module* argument can be used to record the module in which the new @@ -364,6 +432,7 @@ The following functions and structs are used to create .. versionadded:: 3.12 + .. c:function:: PyObject* PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromMetaclass(NULL, module, spec, bases)``. @@ -390,6 +459,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, bases)``. @@ -411,6 +481,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, NULL)``. @@ -431,6 +502,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: int PyType_Freeze(PyTypeObject *type) Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag. @@ -539,9 +611,9 @@ The following functions and structs are used to create :c:type:`PyAsyncMethods` with an added ``Py_`` prefix. For example, use: - * ``Py_tp_dealloc`` to set :c:member:`PyTypeObject.tp_dealloc` - * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` - * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` + * :c:data:`Py_tp_dealloc` to set :c:member:`PyTypeObject.tp_dealloc` + * :c:data:`Py_nb_add` to set :c:member:`PyNumberMethods.nb_add` + * :c:data:`Py_sq_length` to set :c:member:`PySequenceMethods.sq_length` An additional slot is supported that does not correspond to a :c:type:`!PyTypeObject` struct field: @@ -560,7 +632,7 @@ The following functions and structs are used to create If it is not possible to switch to a ``MANAGED`` flag (for example, for vectorcall or to support Python older than 3.12), specify the - offset in :c:member:`Py_tp_members `. + offset in :c:data:`Py_tp_members`. See :ref:`PyMemberDef documentation ` for details. @@ -587,8 +659,8 @@ The following functions and structs are used to create under the :ref:`limited API `. .. versionchanged:: 3.14 - The field :c:member:`~PyTypeObject.tp_vectorcall` can now set - using ``Py_tp_vectorcall``. See the field's documentation + The field :c:member:`~PyTypeObject.tp_vectorcall` can now be set + using :c:data:`Py_tp_vectorcall`. See the field's documentation for details. .. c:member:: void *pfunc @@ -598,10 +670,11 @@ The following functions and structs are used to create *pfunc* values may not be ``NULL``, except for the following slots: - * ``Py_tp_doc`` + * :c:data:`Py_tp_doc` * :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` rather than ``NULL``) + .. c:macro:: Py_tp_token A :c:member:`~PyType_Slot.slot` that records a static memory layout ID diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 8bd3144f88a..d3d8239365f 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -676,6 +676,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_dealloc + .. corresponding-type-slot:: Py_tp_dealloc + A pointer to the instance destructor function. The function signature is:: void tp_dealloc(PyObject *self); @@ -860,6 +862,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getattrfunc PyTypeObject.tp_getattr + .. corresponding-type-slot:: Py_tp_getattr + An optional pointer to the get-attribute-string function. This field is deprecated. When it is defined, it should point to a function @@ -877,6 +881,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: setattrfunc PyTypeObject.tp_setattr + .. corresponding-type-slot:: Py_tp_setattr + An optional pointer to the function for setting and deleting attributes. This field is deprecated. When it is defined, it should point to a function @@ -909,6 +915,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: reprfunc PyTypeObject.tp_repr + .. corresponding-type-slot:: Py_tp_repr + .. index:: pair: built-in function; repr An optional pointer to a function that implements the built-in function @@ -974,6 +982,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: hashfunc PyTypeObject.tp_hash + .. corresponding-type-slot:: Py_tp_hash + .. index:: pair: built-in function; hash An optional pointer to a function that implements the built-in function @@ -1015,6 +1025,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: ternaryfunc PyTypeObject.tp_call + .. corresponding-type-slot:: Py_tp_call + An optional pointer to a function that implements calling the object. This should be ``NULL`` if the object is not callable. The signature is the same as for :c:func:`PyObject_Call`:: @@ -1028,6 +1040,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: reprfunc PyTypeObject.tp_str + .. corresponding-type-slot:: Py_tp_str + An optional pointer to a function that implements the built-in operation :func:`str`. (Note that :class:`str` is a type now, and :func:`str` calls the constructor for that type. This constructor calls :c:func:`PyObject_Str` to do @@ -1053,6 +1067,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getattrofunc PyTypeObject.tp_getattro + .. corresponding-type-slot:: Py_tp_getattro + An optional pointer to the get-attribute function. The signature is the same as for :c:func:`PyObject_GetAttr`:: @@ -1077,6 +1093,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: setattrofunc PyTypeObject.tp_setattro + .. corresponding-type-slot:: Py_tp_setattro + An optional pointer to the function for setting and deleting attributes. The signature is the same as for :c:func:`PyObject_SetAttr`:: @@ -1260,7 +1278,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) This bit indicates that instances of the class have a :attr:`~object.__dict__` attribute, and that the space for the dictionary is managed by the VM. - If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` should also be set. + If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` must also be set. The type traverse function must call :c:func:`PyObject_VisitManagedDict` and its clear function must call :c:func:`PyObject_ClearManagedDict`. @@ -1278,6 +1296,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) This bit indicates that instances of the class should be weakly referenceable. + If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` must also be set. + .. versionadded:: 3.12 **Inheritance:** @@ -1331,8 +1351,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_BASE_EXC_SUBCLASS .. c:macro:: Py_TPFLAGS_TYPE_SUBCLASS - These flags are used by functions such as - :c:func:`PyLong_Check` to quickly determine if a type is a subclass + Functions such as :c:func:`PyLong_Check` will call :c:func:`PyType_FastSubclass` + with one of these flags to quickly determine if a type is a subclass of a built-in type; such specific checks are faster than a generic check, like :c:func:`PyObject_IsInstance`. Custom types that inherit from built-ins should have their :c:member:`~PyTypeObject.tp_flags` @@ -1353,6 +1373,9 @@ and :c:data:`PyType_Type` effectively act as defaults.) type structure. + .. c:macro:: _Py_TPFLAGS_HAVE_VECTORCALL + :no-typesetting: + .. c:macro:: Py_TPFLAGS_HAVE_VECTORCALL This bit is set when the class implements @@ -1364,7 +1387,12 @@ and :c:data:`PyType_Type` effectively act as defaults.) This bit is inherited if :c:member:`~PyTypeObject.tp_call` is also inherited. - .. versionadded:: 3.9 + .. versionadded:: 3.8 as ``_Py_TPFLAGS_HAVE_VECTORCALL`` + + .. versionchanged:: 3.9 + + Renamed to the current name, without the leading underscore. + The old provisional name is :term:`soft deprecated`. .. versionchanged:: 3.12 @@ -1471,8 +1499,58 @@ and :c:data:`PyType_Type` effectively act as defaults.) It will be removed in a future version of CPython + .. c:macro:: Py_TPFLAGS_HAVE_VERSION_TAG + + This macro does nothing. + Historically, this would indicate that the + :c:member:`~PyTypeObject.tp_version_tag` field was available and + initialized. + + .. soft-deprecated:: 3.13 + + + .. c:macro:: Py_TPFLAGS_INLINE_VALUES + + This bit indicates that instances of this type will have an "inline values" + array (containing the object's attributes) placed directly after the end + of the object. + + This requires that :c:macro:`Py_TPFLAGS_HAVE_GC` is set. + + **Inheritance:** + + This flag is not inherited. + + .. versionadded:: 3.13 + + + .. c:macro:: Py_TPFLAGS_IS_ABSTRACT + + This bit indicates that this is an abstract type and therefore cannot + be instantiated. + + **Inheritance:** + + This flag is not inherited. + + .. seealso:: + :mod:`abc` + + + .. c:macro:: Py_TPFLAGS_HAVE_STACKLESS_EXTENSION + + Internal. Do not set or unset this flag. + Historically, this was a reserved flag for use in Stackless Python. + + .. warning:: + This flag is present in header files, but is not be used. + This may be removed in a future version of CPython. + + .. c:member:: const char* PyTypeObject.tp_doc + .. corresponding-type-slot:: Py_tp_doc + An optional pointer to a NUL-terminated C string giving the docstring for this type object. This is exposed as the :attr:`~type.__doc__` attribute on the type and instances of the type. @@ -1484,89 +1562,12 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: traverseproc PyTypeObject.tp_traverse + .. corresponding-type-slot:: Py_tp_traverse + An optional pointer to a traversal function for the garbage collector. This is - only used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is:: + only used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. - int tp_traverse(PyObject *self, visitproc visit, void *arg); - - More information about Python's garbage collection scheme can be found - in section :ref:`supporting-cycle-detection`. - - The :c:member:`~PyTypeObject.tp_traverse` pointer is used by the garbage collector to detect - reference cycles. A typical implementation of a :c:member:`~PyTypeObject.tp_traverse` function - simply calls :c:func:`Py_VISIT` on each of the instance's members that are Python - objects that the instance owns. For example, this is function :c:func:`!local_traverse` from the - :mod:`!_thread` extension module:: - - static int - local_traverse(PyObject *op, visitproc visit, void *arg) - { - localobject *self = (localobject *) op; - Py_VISIT(self->args); - Py_VISIT(self->kw); - Py_VISIT(self->dict); - return 0; - } - - Note that :c:func:`Py_VISIT` is called only on those members that can participate - in reference cycles. Although there is also a ``self->key`` member, it can only - be ``NULL`` or a Python string and therefore cannot be part of a reference cycle. - - On the other hand, even if you know a member can never be part of a cycle, as a - debugging aid you may want to visit it anyway just so the :mod:`gc` module's - :func:`~gc.get_referents` function will include it. - - Heap types (:c:macro:`Py_TPFLAGS_HEAPTYPE`) must visit their type with:: - - Py_VISIT(Py_TYPE(self)); - - It is only needed since Python 3.9. To support Python 3.8 and older, this - line must be conditional:: - - #if PY_VERSION_HEX >= 0x03090000 - Py_VISIT(Py_TYPE(self)); - #endif - - If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the - :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call - :c:func:`PyObject_VisitManagedDict` like this:: - - PyObject_VisitManagedDict((PyObject*)self, visit, arg); - - .. warning:: - When implementing :c:member:`~PyTypeObject.tp_traverse`, only the - members that the instance *owns* (by having :term:`strong references - ` to them) must be - visited. For instance, if an object supports weak references via the - :c:member:`~PyTypeObject.tp_weaklist` slot, the pointer supporting - the linked list (what *tp_weaklist* points to) must **not** be - visited as the instance does not directly own the weak references to itself - (the weakreference list is there to support the weak reference machinery, - but the instance has no strong reference to the elements inside it, as they - are allowed to be removed even if the instance is still alive). - - Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to - :c:func:`!local_traverse` to have these specific names; don't name them just - anything. - - Instances of :ref:`heap-allocated types ` hold a reference to - their type. Their traversal function must therefore either visit - :c:func:`Py_TYPE(self) `, or delegate this responsibility by - calling ``tp_traverse`` of another heap-allocated type (such as a - heap-allocated superclass). - If they do not, the type object may not be garbage-collected. - - .. note:: - - The :c:member:`~PyTypeObject.tp_traverse` function can be called from any - thread. - - .. versionchanged:: 3.9 - - Heap-allocated types are expected to visit ``Py_TYPE(self)`` in - ``tp_traverse``. In earlier versions of Python, due to - `bug 40217 `_, doing this - may lead to crashes in subclasses. + See :ref:`gc-traversal` for documentation. **Inheritance:** @@ -1580,6 +1581,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: inquiry PyTypeObject.tp_clear + .. corresponding-type-slot:: Py_tp_clear + An optional pointer to a clear function. The signature is:: int tp_clear(PyObject *); @@ -1704,7 +1707,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:func:`Py_CLEAR` macro performs the operations in a safe order. If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the - :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call + :c:member:`~PyTypeObject.tp_flags` field, the clear function must call :c:func:`PyObject_ClearManagedDict` like this:: PyObject_ClearManagedDict((PyObject*)self); @@ -1728,6 +1731,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: richcmpfunc PyTypeObject.tp_richcompare + .. corresponding-type-slot:: Py_tp_richcompare + An optional pointer to the rich comparison function, whose signature is:: PyObject *tp_richcompare(PyObject *self, PyObject *other, int op); @@ -1830,6 +1835,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getiterfunc PyTypeObject.tp_iter + .. corresponding-type-slot:: Py_tp_iter + An optional pointer to a function that returns an :term:`iterator` for the object. Its presence normally signals that the instances of this type are :term:`iterable` (although sequences may be iterable without this function). @@ -1845,6 +1852,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: iternextfunc PyTypeObject.tp_iternext + .. corresponding-type-slot:: Py_tp_iternext + An optional pointer to a function that returns the next item in an :term:`iterator`. The signature is:: @@ -1868,6 +1877,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyMethodDef* PyTypeObject.tp_methods + .. corresponding-type-slot:: Py_tp_methods + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMethodDef` structures, declaring regular methods of this type. @@ -1882,6 +1893,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyMemberDef* PyTypeObject.tp_members + .. corresponding-type-slot:: Py_tp_members + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMemberDef` structures, declaring regular data members (fields or slots) of instances of this type. @@ -1897,6 +1910,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyGetSetDef* PyTypeObject.tp_getset + .. corresponding-type-slot:: Py_tp_getset + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyGetSetDef` structures, declaring computed attributes of instances of this type. @@ -1911,6 +1926,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: PyTypeObject* PyTypeObject.tp_base + .. corresponding-type-slot:: Py_tp_base + An optional pointer to a base type from which type properties are inherited. At this level, only single inheritance is supported; multiple inheritance require dynamically creating a type object by calling the metatype. @@ -1983,6 +2000,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: descrgetfunc PyTypeObject.tp_descr_get + .. corresponding-type-slot:: Py_tp_descr_get + An optional pointer to a "descriptor get" function. The function signature is:: @@ -1998,6 +2017,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: descrsetfunc PyTypeObject.tp_descr_set + .. corresponding-type-slot:: Py_tp_descr_set + An optional pointer to a function for setting and deleting a descriptor's value. @@ -2058,6 +2079,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: initproc PyTypeObject.tp_init + .. corresponding-type-slot:: Py_tp_init + An optional pointer to an instance initialization function. This function corresponds to the :meth:`~object.__init__` method of classes. Like @@ -2093,6 +2116,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: allocfunc PyTypeObject.tp_alloc + .. corresponding-type-slot:: Py_tp_alloc + An optional pointer to an instance allocation function. The function signature is:: @@ -2116,6 +2141,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: newfunc PyTypeObject.tp_new + .. corresponding-type-slot:: Py_tp_new + An optional pointer to an instance creation function. The function signature is:: @@ -2155,6 +2182,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: freefunc PyTypeObject.tp_free + .. corresponding-type-slot:: Py_tp_free + An optional pointer to an instance deallocation function. Its signature is:: void tp_free(void *self); @@ -2184,6 +2213,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: inquiry PyTypeObject.tp_is_gc + .. corresponding-type-slot:: Py_tp_is_gc + An optional pointer to a function called by the garbage collector. The garbage collector needs to know whether a particular object is collectible @@ -2212,12 +2243,14 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: PyObject* PyTypeObject.tp_bases + .. corresponding-type-slot:: Py_tp_bases + Tuple of base types. This field should be set to ``NULL`` and treated as read-only. Python will fill it in when the type is :c:func:`initialized `. - For dynamically created classes, the ``Py_tp_bases`` + For dynamically created classes, the :c:data:`Py_tp_bases` :c:type:`slot ` can be used instead of the *bases* argument of :c:func:`PyType_FromSpecWithBases`. The argument form is preferred. @@ -2292,6 +2325,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_del + .. corresponding-type-slot:: Py_tp_del + This field is deprecated. Use :c:member:`~PyTypeObject.tp_finalize` instead. @@ -2306,6 +2341,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_finalize + .. corresponding-type-slot:: Py_tp_finalize + An optional pointer to an instance finalization function. This is the C implementation of the :meth:`~object.__del__` special method. Its signature is:: @@ -2464,6 +2501,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: vectorcallfunc PyTypeObject.tp_vectorcall + .. corresponding-type-slot:: Py_tp_vectorcall + A :ref:`vectorcall function ` to use for calls of this type object (rather than instances). In other words, ``tp_vectorcall`` can be used to optimize ``type.__call__``, @@ -2557,9 +2596,6 @@ This is done by filling a :c:type:`PyType_Spec` structure and calling Number Object Structures ------------------------ -.. sectionauthor:: Amaury Forgeot d'Arc - - .. c:type:: PyNumberMethods This structure holds pointers to the functions which an object uses to @@ -2629,51 +2665,154 @@ Number Object Structures Python 3.0.1. .. c:member:: binaryfunc PyNumberMethods.nb_add + + .. corresponding-type-slot:: Py_nb_add + .. c:member:: binaryfunc PyNumberMethods.nb_subtract + + .. corresponding-type-slot:: Py_nb_subtract + .. c:member:: binaryfunc PyNumberMethods.nb_multiply + + .. corresponding-type-slot:: Py_nb_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_remainder + + .. corresponding-type-slot:: Py_nb_remainder + .. c:member:: binaryfunc PyNumberMethods.nb_divmod + + .. corresponding-type-slot:: Py_nb_divmod + .. c:member:: ternaryfunc PyNumberMethods.nb_power + + .. corresponding-type-slot:: Py_nb_power + .. c:member:: unaryfunc PyNumberMethods.nb_negative + + .. corresponding-type-slot:: Py_nb_negative + .. c:member:: unaryfunc PyNumberMethods.nb_positive + + .. corresponding-type-slot:: Py_nb_positive + .. c:member:: unaryfunc PyNumberMethods.nb_absolute + + .. corresponding-type-slot:: Py_nb_absolute + .. c:member:: inquiry PyNumberMethods.nb_bool + + .. corresponding-type-slot:: Py_nb_bool + .. c:member:: unaryfunc PyNumberMethods.nb_invert + + .. corresponding-type-slot:: Py_nb_invert + .. c:member:: binaryfunc PyNumberMethods.nb_lshift + + .. corresponding-type-slot:: Py_nb_lshift + .. c:member:: binaryfunc PyNumberMethods.nb_rshift + + .. corresponding-type-slot:: Py_nb_rshift + .. c:member:: binaryfunc PyNumberMethods.nb_and + + .. corresponding-type-slot:: Py_nb_and + .. c:member:: binaryfunc PyNumberMethods.nb_xor + + .. corresponding-type-slot:: Py_nb_xor + .. c:member:: binaryfunc PyNumberMethods.nb_or + + .. corresponding-type-slot:: Py_nb_or + .. c:member:: unaryfunc PyNumberMethods.nb_int + + .. corresponding-type-slot:: Py_nb_int + .. c:member:: void *PyNumberMethods.nb_reserved + .. c:member:: unaryfunc PyNumberMethods.nb_float + + .. corresponding-type-slot:: Py_nb_float + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_add + + .. corresponding-type-slot:: Py_nb_inplace_add + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_subtract + + .. corresponding-type-slot:: Py_nb_inplace_subtract + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_multiply + + .. corresponding-type-slot:: Py_nb_inplace_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_remainder + + .. corresponding-type-slot:: Py_nb_inplace_remainder + .. c:member:: ternaryfunc PyNumberMethods.nb_inplace_power + + .. corresponding-type-slot:: Py_nb_inplace_power + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_lshift + + .. corresponding-type-slot:: Py_nb_inplace_lshift + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_rshift + + .. corresponding-type-slot:: Py_nb_inplace_rshift + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_and + + .. corresponding-type-slot:: Py_nb_inplace_and + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_xor + + .. corresponding-type-slot:: Py_nb_inplace_xor + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_or + + .. corresponding-type-slot:: Py_nb_inplace_or + .. c:member:: binaryfunc PyNumberMethods.nb_floor_divide + + .. corresponding-type-slot:: Py_nb_floor_divide + .. c:member:: binaryfunc PyNumberMethods.nb_true_divide + + .. corresponding-type-slot:: Py_nb_true_divide + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_floor_divide + + .. corresponding-type-slot:: Py_nb_inplace_floor_divide + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_true_divide + + .. corresponding-type-slot:: Py_nb_inplace_true_divide + .. c:member:: unaryfunc PyNumberMethods.nb_index + + .. corresponding-type-slot:: Py_nb_index + .. c:member:: binaryfunc PyNumberMethods.nb_matrix_multiply + + .. corresponding-type-slot:: Py_nb_matrix_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_matrix_multiply + .. corresponding-type-slot:: Py_nb_inplace_matrix_multiply + + .. _mapping-structs: Mapping Object Structures ------------------------- -.. sectionauthor:: Amaury Forgeot d'Arc - - .. c:type:: PyMappingMethods This structure holds pointers to the functions which an object uses to @@ -2681,12 +2820,16 @@ Mapping Object Structures .. c:member:: lenfunc PyMappingMethods.mp_length + .. corresponding-type-slot:: Py_mp_length + This function is used by :c:func:`PyMapping_Size` and :c:func:`PyObject_Size`, and has the same signature. This slot may be set to ``NULL`` if the object has no defined length. .. c:member:: binaryfunc PyMappingMethods.mp_subscript + .. corresponding-type-slot:: Py_mp_subscript + This function is used by :c:func:`PyObject_GetItem` and :c:func:`PySequence_GetSlice`, and has the same signature as :c:func:`!PyObject_GetItem`. This slot must be filled for the @@ -2695,6 +2838,8 @@ Mapping Object Structures .. c:member:: objobjargproc PyMappingMethods.mp_ass_subscript + .. corresponding-type-slot:: Py_mp_ass_subscript + This function is used by :c:func:`PyObject_SetItem`, :c:func:`PyObject_DelItem`, :c:func:`PySequence_SetSlice` and :c:func:`PySequence_DelSlice`. It has the same signature as @@ -2708,9 +2853,6 @@ Mapping Object Structures Sequence Object Structures -------------------------- -.. sectionauthor:: Amaury Forgeot d'Arc - - .. c:type:: PySequenceMethods This structure holds pointers to the functions which an object uses to @@ -2718,6 +2860,8 @@ Sequence Object Structures .. c:member:: lenfunc PySequenceMethods.sq_length + .. corresponding-type-slot:: Py_sq_length + This function is used by :c:func:`PySequence_Size` and :c:func:`PyObject_Size`, and has the same signature. It is also used for handling negative indices via the :c:member:`~PySequenceMethods.sq_item` @@ -2725,18 +2869,24 @@ Sequence Object Structures .. c:member:: binaryfunc PySequenceMethods.sq_concat + .. corresponding-type-slot:: Py_sq_concat + This function is used by :c:func:`PySequence_Concat` and has the same signature. It is also used by the ``+`` operator, after trying the numeric addition via the :c:member:`~PyNumberMethods.nb_add` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_repeat + .. corresponding-type-slot:: Py_sq_repeat + This function is used by :c:func:`PySequence_Repeat` and has the same signature. It is also used by the ``*`` operator, after trying numeric multiplication via the :c:member:`~PyNumberMethods.nb_multiply` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_item + .. corresponding-type-slot:: Py_sq_item + This function is used by :c:func:`PySequence_GetItem` and has the same signature. It is also used by :c:func:`PyObject_GetItem`, after trying the subscription via the :c:member:`~PyMappingMethods.mp_subscript` slot. @@ -2750,6 +2900,8 @@ Sequence Object Structures .. c:member:: ssizeobjargproc PySequenceMethods.sq_ass_item + .. corresponding-type-slot:: Py_sq_ass_item + This function is used by :c:func:`PySequence_SetItem` and has the same signature. It is also used by :c:func:`PyObject_SetItem` and :c:func:`PyObject_DelItem`, after trying the item assignment and deletion @@ -2759,6 +2911,8 @@ Sequence Object Structures .. c:member:: objobjproc PySequenceMethods.sq_contains + .. corresponding-type-slot:: Py_sq_contains + This function may be used by :c:func:`PySequence_Contains` and has the same signature. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_Contains` simply traverses the sequence until it @@ -2766,6 +2920,8 @@ Sequence Object Structures .. c:member:: binaryfunc PySequenceMethods.sq_inplace_concat + .. corresponding-type-slot:: Py_sq_inplace_concat + This function is used by :c:func:`PySequence_InPlaceConcat` and has the same signature. It should modify its first operand, and return it. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceConcat` @@ -2775,6 +2931,8 @@ Sequence Object Structures .. c:member:: ssizeargfunc PySequenceMethods.sq_inplace_repeat + .. corresponding-type-slot:: Py_sq_inplace_repeat + This function is used by :c:func:`PySequence_InPlaceRepeat` and has the same signature. It should modify its first operand, and return it. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceRepeat` @@ -2788,10 +2946,6 @@ Sequence Object Structures Buffer Object Structures ------------------------ -.. sectionauthor:: Greg J. Stein -.. sectionauthor:: Benjamin Peterson -.. sectionauthor:: Stefan Krah - .. c:type:: PyBufferProcs This structure holds pointers to the functions required by the @@ -2800,6 +2954,8 @@ Buffer Object Structures .. c:member:: getbufferproc PyBufferProcs.bf_getbuffer + .. corresponding-type-slot:: Py_bf_getbuffer + The signature of this function is:: int (PyObject *exporter, Py_buffer *view, int flags); @@ -2819,6 +2975,24 @@ Buffer Object Structures (5) Return ``0``. + **Thread safety:** + + In the :term:`free-threaded build`, implementations must ensure: + + * The export counter increment in step (3) is atomic. + + * The underlying buffer data remains valid and at a stable memory + location for the lifetime of all exports. + + * For objects that support resizing or reallocation (such as + :class:`bytearray`), the export counter is checked atomically before + such operations, and :exc:`BufferError` is raised if exports exist. + + * The function is safe to call concurrently from multiple threads. + + See also :ref:`thread-safety-memoryview` for the Python-level + thread safety guarantees of :class:`memoryview` objects. + If *exporter* is part of a chain or tree of buffer providers, two main schemes can be used: @@ -2849,6 +3023,8 @@ Buffer Object Structures .. c:member:: releasebufferproc PyBufferProcs.bf_releasebuffer + .. corresponding-type-slot:: Py_bf_releasebuffer + The signature of this function is:: void (PyObject *exporter, Py_buffer *view); @@ -2862,6 +3038,16 @@ Buffer Object Structures (2) If the counter is ``0``, free all memory associated with *view*. + **Thread safety:** + + In the :term:`free-threaded build`: + + * The export counter decrement in step (1) must be atomic. + + * Resource cleanup when the counter reaches zero must be done atomically, + as the final release may race with concurrent releases from other + threads and dellocation must only happen once. + The exporter MUST use the :c:member:`~Py_buffer.internal` field to keep track of buffer-specific resources. This field is guaranteed to remain constant, while a consumer MAY pass a copy of the original buffer as the @@ -2883,8 +3069,6 @@ Buffer Object Structures Async Object Structures ----------------------- -.. sectionauthor:: Yury Selivanov - .. versionadded:: 3.5 .. c:type:: PyAsyncMethods @@ -2903,6 +3087,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_await + .. corresponding-type-slot:: Py_am_await + The signature of this function is:: PyObject *am_await(PyObject *self); @@ -2914,6 +3100,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_aiter + .. corresponding-type-slot:: Py_am_aiter + The signature of this function is:: PyObject *am_aiter(PyObject *self); @@ -2926,6 +3114,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_anext + .. corresponding-type-slot:: Py_am_anext + The signature of this function is:: PyObject *am_anext(PyObject *self); @@ -2936,6 +3126,8 @@ Async Object Structures .. c:member:: sendfunc PyAsyncMethods.am_send + .. corresponding-type-slot:: Py_am_send + The signature of this function is:: PySendResult am_send(PyObject *self, PyObject *arg, PyObject **result); diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 84fee05cb4c..059a7ef399a 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -5,9 +5,6 @@ Unicode Objects and Codecs -------------------------- -.. sectionauthor:: Marc-André Lemburg -.. sectionauthor:: Georg Brandl - Unicode Objects ^^^^^^^^^^^^^^^ @@ -65,6 +62,27 @@ Python: .. versionadded:: 3.3 + The structure of a particular object can be determined using the following + macros. + The macros cannot fail; their behavior is undefined if their argument + is not a Python Unicode object. + + .. c:namespace:: NULL + + .. c:macro:: PyUnicode_IS_COMPACT(o) + + True if *o* uses the :c:struct:`PyCompactUnicodeObject` structure. + + .. versionadded:: 3.3 + + + .. c:macro:: PyUnicode_IS_COMPACT_ASCII(o) + + True if *o* uses the :c:struct:`PyASCIIObject` structure. + + .. versionadded:: 3.3 + + The following APIs are C macros and static inlined functions for fast checks and access to internal read-only data of Unicode objects: @@ -321,12 +339,22 @@ These APIs can be used to work with surrogates: Check if *ch* is a low surrogate (``0xDC00 <= ch <= 0xDFFF``). +.. c:function:: Py_UCS4 Py_UNICODE_HIGH_SURROGATE(Py_UCS4 ch) + + Return the high UTF-16 surrogate (``0xD800`` to ``0xDBFF``) for a Unicode + code point in the range ``[0x10000; 0x10FFFF]``. + +.. c:function:: Py_UCS4 Py_UNICODE_LOW_SURROGATE(Py_UCS4 ch) + + Return the low UTF-16 surrogate (``0xDC00`` to ``0xDFFF``) for a Unicode + code point in the range ``[0x10000; 0x10FFFF]``. + .. c:function:: Py_UCS4 Py_UNICODE_JOIN_SURROGATES(Py_UCS4 high, Py_UCS4 low) Join two surrogate code points and return a single :c:type:`Py_UCS4` value. *high* and *low* are respectively the leading and trailing surrogates in a - surrogate pair. *high* must be in the range [0xD800; 0xDBFF] and *low* must - be in the range [0xDC00; 0xDFFF]. + surrogate pair. *high* must be in the range ``[0xD800; 0xDBFF]`` and *low* must + be in the range ``[0xDC00; 0xDFFF]``. Creating and accessing Unicode strings @@ -747,7 +775,7 @@ APIs: Return ``0`` on success, ``-1`` on error with an exception set. This function checks that *unicode* is a Unicode object, that the index is - not out of bounds, and that the object's reference count is one). + not out of bounds, and that the object's reference count is one. See :c:func:`PyUnicode_WRITE` for a version that skips these checks, making them your responsibility. @@ -1827,8 +1855,6 @@ object. On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. - .. versionadded:: 3.14 - .. c:function:: int PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, const wchar_t *str, Py_ssize_t size) Write the wide string *str* into *writer*. @@ -1839,7 +1865,7 @@ object. On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. -.. c:function:: int PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *writer, Py_UCS4 *str, Py_ssize_t size) +.. c:function:: int PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *writer, const Py_UCS4 *str, Py_ssize_t size) Writer the UCS4 string *str* into *writer*. @@ -1855,13 +1881,23 @@ object. On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. + To write a :class:`str` subclass which overrides the :meth:`~object.__str__` + method, :c:func:`PyUnicode_FromObject` can be used to get the original + string. + .. c:function:: int PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) Call :c:func:`PyObject_Repr` on *obj* and write the output into *writer*. + If *obj* is ``NULL``, write the string ``""`` into *writer*. + On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. + .. versionchanged:: 3.14.4 + + Added support for ``NULL``. + .. c:function:: int PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, Py_ssize_t start, Py_ssize_t end) Write the substring ``str[start:end]`` into *writer*. diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index fb07fec7eff..6256bf7a145 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -13,8 +13,9 @@ the interpreter. Several of these functions accept a start symbol from the grammar as a parameter. The available start symbols are :c:data:`Py_eval_input`, -:c:data:`Py_file_input`, and :c:data:`Py_single_input`. These are described -following the functions which accept them as parameters. +:c:data:`Py_file_input`, :c:data:`Py_single_input`, and +:c:data:`Py_func_type_input`. These are described following the functions +which accept them as parameters. Note also that several of these functions take :c:expr:`FILE*` parameters. One particular issue which needs to be handled carefully is that the :c:type:`FILE` @@ -99,6 +100,20 @@ the same library that the Python runtime is using. Otherwise, Python may not handle script file with LF line ending correctly. +.. c:function:: int PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) + + Read and execute a single statement from a file associated with an + interactive device according to the *flags* argument. The user will be + prompted using ``sys.ps1`` and ``sys.ps2``. *filename* must be a Python + :class:`str` object. + + Returns ``0`` when the input was + executed successfully, ``-1`` if there was an exception, or an error code + from the :file:`errcode.h` include file distributed as part of Python if + there was a parse error. (Note that :file:`errcode.h` is not included by + :file:`Python.h`, so must be included specifically if needed.) + + .. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, @@ -107,17 +122,10 @@ the same library that the Python runtime is using. .. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) - Read and execute a single statement from a file associated with an - interactive device according to the *flags* argument. The user will be - prompted using ``sys.ps1`` and ``sys.ps2``. *filename* is decoded from the + Similar to :c:func:`PyRun_InteractiveOneObject`, but *filename* is a + :c:expr:`const char*`, which is decoded from the :term:`filesystem encoding and error handler`. - Returns ``0`` when the input was - executed successfully, ``-1`` if there was an exception, or an error code - from the :file:`errcode.h` include file distributed as part of Python if - there was a parse error. (Note that :file:`errcode.h` is not included by - :file:`Python.h`, so must be included specifically if needed.) - .. c:function:: int PyRun_InteractiveLoop(FILE *fp, const char *filename) @@ -140,7 +148,7 @@ the same library that the Python runtime is using. interpreter prompt is about to become idle and wait for user input from the terminal. The return value is ignored. Overriding this hook can be used to integrate the interpreter's prompt with other - event loops, as done in the :file:`Modules/_tkinter.c` in the + event loops, as done in :file:`Modules/_tkinter.c` in the Python source code. .. versionchanged:: 3.12 @@ -183,7 +191,7 @@ the same library that the Python runtime is using. objects *globals* and *locals* with the compiler flags specified by *flags*. *globals* must be a dictionary; *locals* can be any object that implements the mapping protocol. The parameter *start* specifies - the start token that should be used to parse the source code. + the start symbol and must be one of the :ref:`available start symbols `. Returns the result of executing the code as a Python object, or ``NULL`` if an exception was raised. @@ -231,9 +239,9 @@ the same library that the Python runtime is using. .. c:function:: PyObject* Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) Parse and compile the Python source code in *str*, returning the resulting code - object. The start token is given by *start*; this can be used to constrain the - code which can be compiled and should be :c:data:`Py_eval_input`, - :c:data:`Py_file_input`, or :c:data:`Py_single_input`. The filename specified by + object. The start symbol is given by *start*; this can be used to constrain the + code which can be compiled and should be :ref:`available start symbols + `. The filename specified by *filename* is used to construct the code object and may appear in tracebacks or :exc:`SyntaxError` exception messages. This returns ``NULL`` if the code cannot be parsed or compiled. @@ -296,32 +304,6 @@ the same library that the Python runtime is using. true on success, false on failure. -.. c:var:: int Py_eval_input - - .. index:: single: Py_CompileString (C function) - - The start symbol from the Python grammar for isolated expressions; for use with - :c:func:`Py_CompileString`. - - -.. c:var:: int Py_file_input - - .. index:: single: Py_CompileString (C function) - - The start symbol from the Python grammar for sequences of statements as read - from a file or other source; for use with :c:func:`Py_CompileString`. This is - the symbol to use when compiling arbitrarily long Python source code. - - -.. c:var:: int Py_single_input - - .. index:: single: Py_CompileString (C function) - - The start symbol from the Python grammar for a single statement; for use with - :c:func:`Py_CompileString`. This is the symbol used for the interactive - interpreter loop. - - .. c:struct:: PyCompilerFlags This is the structure used to hold compiler flags. In cases where code is only @@ -365,3 +347,92 @@ the same library that the Python runtime is using. as :c:macro:`CO_FUTURE_ANNOTATIONS` to enable features normally selectable using :ref:`future statements `. See :ref:`c_codeobject_flags` for a complete list. + + +.. _start-symbols: + +Available start symbols +^^^^^^^^^^^^^^^^^^^^^^^ + + +.. c:var:: int Py_eval_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for isolated expressions; for use with + :c:func:`Py_CompileString`. + + +.. c:var:: int Py_file_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for sequences of statements as read + from a file or other source; for use with :c:func:`Py_CompileString`. This is + the symbol to use when compiling arbitrarily long Python source code. + + +.. c:var:: int Py_single_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for a single statement; for use with + :c:func:`Py_CompileString`. This is the symbol used for the interactive + interpreter loop. + + +.. c:var:: int Py_func_type_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for a function type; for use with + :c:func:`Py_CompileString`. This is used to parse "signature type comments" + from :pep:`484`. + + This requires the :c:macro:`PyCF_ONLY_AST` flag to be set. + + .. seealso:: + * :py:class:`ast.FunctionType` + * :pep:`484` + + .. versionadded:: 3.8 + + +Stack Effects +^^^^^^^^^^^^^ + +.. seealso:: + :py:func:`dis.stack_effect` + + +.. c:macro:: PY_INVALID_STACK_EFFECT + + Sentinel value representing an invalid stack effect. + + This is currently equivalent to ``INT_MAX``. + + .. versionadded:: 3.8 + + +.. c:function:: int PyCompile_OpcodeStackEffect(int opcode, int oparg) + + Compute the stack effect of *opcode* with argument *oparg*. + + On success, this function returns the stack effect; on failure, this + returns :c:macro:`PY_INVALID_STACK_EFFECT`. + + .. versionadded:: 3.4 + + +.. c:function:: int PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump) + + Similar to :c:func:`PyCompile_OpcodeStackEffect`, but don't include the + stack effect of jumping if *jump* is zero. + + If *jump* is ``0``, this will not include the stack effect of jumping, but + if *jump* is ``1`` or ``-1``, this will include it. + + On success, this function returns the stack effect; on failure, this + returns :c:macro:`PY_INVALID_STACK_EFFECT`. + + .. versionadded:: 3.8 diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index c3c6cf413dc..db6ae0a9d4e 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -19,7 +19,14 @@ as much as it can. .. c:function:: int PyWeakref_CheckRef(PyObject *ob) - Return non-zero if *ob* is a reference object. This function always succeeds. + Return non-zero if *ob* is a reference object or a subclass of the reference + type. This function always succeeds. + + +.. c:function:: int PyWeakref_CheckRefExact(PyObject *ob) + + Return non-zero if *ob* is a reference object, but not a subclass of the + reference type. This function always succeeds. .. c:function:: int PyWeakref_CheckProxy(PyObject *ob) @@ -38,6 +45,10 @@ as much as it can. weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + .. seealso:: + :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly + referenceable. + .. c:function:: PyObject* PyWeakref_NewProxy(PyObject *ob, PyObject *callback) @@ -50,6 +61,10 @@ as much as it can. is not a weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + .. seealso:: + :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly + referenceable. + .. c:function:: int PyWeakref_GetRef(PyObject *ref, PyObject **pobj) @@ -64,30 +79,6 @@ as much as it can. .. versionadded:: 3.13 -.. c:function:: PyObject* PyWeakref_GetObject(PyObject *ref) - - Return a :term:`borrowed reference` to the referenced object from a weak - reference, *ref*. If the referent is no longer live, returns ``Py_None``. - - .. note:: - - This function returns a :term:`borrowed reference` to the referenced object. - This means that you should always call :c:func:`Py_INCREF` on the object - except when it cannot be destroyed before the last usage of the borrowed - reference. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyWeakref_GetRef` instead. - - -.. c:function:: PyObject* PyWeakref_GET_OBJECT(PyObject *ref) - - Similar to :c:func:`PyWeakref_GetObject`, but does no error checking. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyWeakref_GetRef` instead. - - .. c:function:: int PyWeakref_IsDead(PyObject *ref) Test if the weak reference *ref* is dead. Returns 1 if the reference is diff --git a/Doc/conf.py b/Doc/conf.py index c1b07df08b1..e2dff74538a 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -33,6 +33,7 @@ 'issue_role', 'lexers', 'misc_news', + 'profiling_trace', 'pydoc_topics', 'pyspecific', 'sphinx.ext.coverage', @@ -42,8 +43,10 @@ # Skip if downstream redistributors haven't installed them _OPTIONAL_EXTENSIONS = ( + 'linklint.ext', 'notfound.extension', 'sphinxext.opengraph', + 'sphinxcontrib.rsvgconverter', ) for optional_ext in _OPTIONAL_EXTENSIONS: try: @@ -70,6 +73,7 @@ # General substitutions. project = 'Python' copyright = "2001 Python Software Foundation" +_doc_authors = 'Python documentation authors' # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. @@ -174,6 +178,7 @@ ('c:type', '__int64'), ('c:type', 'unsigned __int64'), ('c:type', 'double'), + ('c:type', '_Float16'), # Standard C structures ('c:struct', 'in6_addr'), ('c:struct', 'in_addr'), @@ -226,13 +231,6 @@ # Temporary undocumented names. # In future this list must be empty. nitpick_ignore += [ - # Undocumented public C macros - ('c:macro', 'Py_BUILD_ASSERT'), - ('c:macro', 'Py_BUILD_ASSERT_EXPR'), - # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot - # be resolved, as the method is currently undocumented. For context, see - # https://github.com/python/cpython/pull/103289. - ('py:meth', '_SubParsersAction.add_parser'), # Attributes/methods/etc. that definitely should be documented better, # but are deferred for now: ('py:attr', '__wrapped__'), @@ -359,73 +357,79 @@ 'papersize': 'a4paper', # The font size ('10pt', '11pt' or '12pt'). 'pointsize': '10pt', + 'maxlistdepth': '8', # See https://github.com/python/cpython/issues/139588 } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). -_stdauthor = 'Guido van Rossum and the Python development team' latex_documents = [ - ('c-api/index', 'c-api.tex', 'The Python/C API', _stdauthor, 'manual'), + ('c-api/index', 'c-api.tex', 'The Python/C API', _doc_authors, 'manual'), ( 'extending/index', 'extending.tex', 'Extending and Embedding Python', - _stdauthor, + _doc_authors, 'manual', ), ( 'installing/index', 'installing.tex', 'Installing Python Modules', - _stdauthor, + _doc_authors, 'manual', ), ( 'library/index', 'library.tex', 'The Python Library Reference', - _stdauthor, + _doc_authors, 'manual', ), ( 'reference/index', 'reference.tex', 'The Python Language Reference', - _stdauthor, + _doc_authors, 'manual', ), ( 'tutorial/index', 'tutorial.tex', 'Python Tutorial', - _stdauthor, + _doc_authors, 'manual', ), ( 'using/index', 'using.tex', 'Python Setup and Usage', - _stdauthor, + _doc_authors, 'manual', ), ( 'faq/index', 'faq.tex', 'Python Frequently Asked Questions', - _stdauthor, + _doc_authors, 'manual', ), ( 'whatsnew/' + version, 'whatsnew.tex', 'What\'s New in Python', - 'A. M. Kuchling', + _doc_authors, 'howto', ), ] # Collect all HOWTOs individually latex_documents.extend( - ('howto/' + fn[:-4], 'howto-' + fn[:-4] + '.tex', '', _stdauthor, 'howto') + ( + 'howto/' + fn[:-4], + 'howto-' + fn[:-4] + '.tex', + '', + _doc_authors, + 'howto', + ) for fn in os.listdir('howto') if fn.endswith('.rst') and fn != 'index.rst' ) @@ -436,14 +440,38 @@ # Options for Epub output # ----------------------- -epub_author = 'Python Documentation Authors' +epub_author = _doc_authors epub_publisher = 'Python Software Foundation' -epub_exclude_files = ('index.xhtml', 'download.xhtml') +epub_exclude_files = ( + 'index.xhtml', + 'download.xhtml', + '_static/tachyon-example-flamegraph.html', + '_static/tachyon-example-heatmap.html', +) # index pages are not valid xhtml # https://github.com/sphinx-doc/sphinx/issues/12359 epub_use_index = False +# translation tag +# --------------- + +language_code = None +for arg in sys.argv: + if arg.startswith('language='): + language_code = arg.split('=', 1)[1] + +if language_code: + tags.add('translation') # noqa: F821 + + rst_epilog += f"""\ +.. _TRANSLATION_REPO: https://github.com/python/python-docs-{language_code.replace("_", "-").lower()} +""" # noqa: F821 +else: + rst_epilog += """\ +.. _TRANSLATION_REPO: https://github.com/python +""" + # Options for the coverage checker # -------------------------------- @@ -536,6 +564,7 @@ # mapping unique short aliases to a base URL and a prefix. # https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html extlinks = { + "oss-fuzz": ("https://issues.oss-fuzz.com/issues/%s", "#%s"), "pypi": ("https://pypi.org/project/%s/", "%s"), "source": (SOURCE_URI, "%s"), } @@ -547,6 +576,18 @@ # Relative filename of the data files refcount_file = 'data/refcounts.dat' stable_abi_file = 'data/stable_abi.dat' +threadsafety_file = 'data/threadsafety.dat' + +# Options for notfound.extension +# ------------------------------- + +if not os.getenv("READTHEDOCS"): + if language_code: + notfound_urls_prefix = ( + f'/{language_code.replace("_", "-").lower()}/{version}/' + ) + else: + notfound_urls_prefix = f'/{version}/' # Options for sphinxext-opengraph # ------------------------------- diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 144c5608e07..663b79e45ee 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -1141,6 +1141,9 @@ PyInterpreterState_Clear:PyInterpreterState*:interp:: PyInterpreterState_Delete:void::: PyInterpreterState_Delete:PyInterpreterState*:interp:: +PyInterpreterState_GetDict:PyObject*::0: +PyInterpreterState_GetDict:PyInterpreterState*:interp:: + PyInterpreterState_GetID:int64_t::: PyInterpreterState_GetID:PyInterpreterState*:interp:: @@ -1469,6 +1472,9 @@ PyModule_Create2:PyObject*::+1: PyModule_Create2:PyModuleDef*:def:: PyModule_Create2:int:module_api_version:: +PyModule_Exec:int::: +PyModule_ExecDef:PyObject*:module:0: + PyModule_ExecDef:int::: PyModule_ExecDef:PyObject*:module:0: PyModule_ExecDef:PyModuleDef*:def:: @@ -1482,6 +1488,10 @@ PyModule_FromDefAndSpec2:PyModuleDef*:def:: PyModule_FromDefAndSpec2:PyObject*:spec:0: PyModule_FromDefAndSpec2:int:module_api_version:: +PyModule_FromSlotsAndSpec:PyObject*::+1: +PyModule_FromSlotsAndSpec:const PyModuleDef_Slot *:slots:: +PyModule_FromSlotsAndSpec:PyObject*:spec:0: + PyModule_GetDef:PyModuleDef*::0: PyModule_GetDef:PyObject*:module:0: @@ -1503,6 +1513,14 @@ PyModule_GetNameObject:PyObject*:module:0: PyModule_GetState:void*::: PyModule_GetState:PyObject*:module:0: +PyModule_GetStateSize:int::: +PyModule_GetStateSize:PyObject*:module:0: +PyModule_GetToken:Py_ssize_t**:result:: + +PyModule_GetToken:int::: +PyModule_GetToken:PyObject*:module:0: +PyModule_GetToken:void**:result:: + PyModule_New:PyObject*::+1: PyModule_New:char*:name:: @@ -2019,6 +2037,10 @@ PySeqIter_Check:PyObject *:op:0: PySeqIter_New:PyObject*::+1: PySeqIter_New:PyObject*:seq:0: +PySentinel_New:PyObject*::+1: +PySentinel_New:const char*:name:: +PySentinel_New:const char*:module_name:: + PySequence_Check:int::: PySequence_Check:PyObject*:o:0: @@ -2409,6 +2431,20 @@ PyType_GetFlags:PyTypeObject*:type:0: PyType_GetName:PyObject*::+1: PyType_GetName:PyTypeObject*:type:0: +PyType_GetModule:PyObject*::0: +PyType_GetModule:PyTypeObject*:type:0: + +PyType_GetModule_DuringGC:PyObject*::0: +PyType_GetModule_DuringGC:PyTypeObject*:type:0: + +PyType_GetModuleByToken:PyObject*::+1: +PyType_GetModuleByToken:PyTypeObject*:type:0: +PyType_GetModuleByToken:PyModuleDef*:def:: + +PyType_GetModuleByToken_DuringGC:PyObject*::0: +PyType_GetModuleByToken_DuringGC:PyTypeObject*:type:0: +PyType_GetModuleByToken_DuringGC:PyModuleDef*:mod_token:: + PyType_GetModuleByDef:PyObject*::0: PyType_GetModuleByDef:PyTypeObject*:type:0: PyType_GetModuleByDef:PyModuleDef*:def:: @@ -2947,12 +2983,6 @@ PyWeakref_CheckProxy:PyObject*:ob:0: PyWeakref_CheckRef:int::: PyWeakref_CheckRef:PyObject*:ob:0: -PyWeakref_GET_OBJECT:PyObject*::0: -PyWeakref_GET_OBJECT:PyObject*:ref:0: - -PyWeakref_GetObject:PyObject*::0: -PyWeakref_GetObject:PyObject*:ref:0: - PyWeakref_GetRef:int::: PyWeakref_GetRef:PyObject*:ref:0: PyWeakref_GetRef:PyObject**:pobj:+1: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 641f7bb3804..4ae5e999f0b 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,7 +1,21 @@ role,name,added,ifdef_note,struct_abi_kind +macro,METH_CLASS,3.2,, +macro,METH_COEXIST,3.2,, +macro,METH_FASTCALL,3.7,, +macro,METH_METHOD,3.7,, +macro,METH_NOARGS,3.2,, +macro,METH_O,3.2,, +macro,METH_STATIC,3.2,, +macro,METH_VARARGS,3.2,, macro,PY_VECTORCALL_ARGUMENTS_OFFSET,3.12,, type,PyABIInfo,3.15,,full-abi func,PyABIInfo_Check,3.15,, +macro,PyABIInfo_DEFAULT_ABI_VERSION,3.15,, +macro,PyABIInfo_DEFAULT_FLAGS,3.15,, +macro,PyABIInfo_FREETHREADED,3.15,, +macro,PyABIInfo_FREETHREADING_AGNOSTIC,3.15,, +macro,PyABIInfo_GIL,3.15,, +macro,PyABIInfo_STABLE,3.15,, macro,PyABIInfo_VAR,3.15,, func,PyAIter_Check,3.10,, func,PyArg_Parse,3.2,, @@ -11,6 +25,26 @@ func,PyArg_UnpackTuple,3.2,, func,PyArg_VaParse,3.2,, func,PyArg_VaParseTupleAndKeywords,3.2,, func,PyArg_ValidateKeywordArguments,3.2,, +macro,PyBUF_ANY_CONTIGUOUS,3.11,, +macro,PyBUF_CONTIG,3.11,, +macro,PyBUF_CONTIG_RO,3.11,, +macro,PyBUF_C_CONTIGUOUS,3.11,, +macro,PyBUF_FORMAT,3.11,, +macro,PyBUF_FULL,3.11,, +macro,PyBUF_FULL_RO,3.11,, +macro,PyBUF_F_CONTIGUOUS,3.11,, +macro,PyBUF_INDIRECT,3.11,, +macro,PyBUF_MAX_NDIM,3.11,, +macro,PyBUF_ND,3.11,, +macro,PyBUF_READ,3.11,, +macro,PyBUF_RECORDS,3.11,, +macro,PyBUF_RECORDS_RO,3.11,, +macro,PyBUF_SIMPLE,3.11,, +macro,PyBUF_STRIDED,3.11,, +macro,PyBUF_STRIDED_RO,3.11,, +macro,PyBUF_STRIDES,3.11,, +macro,PyBUF_WRITABLE,3.11,, +macro,PyBUF_WRITE,3.11,, data,PyBaseObject_Type,3.2,, func,PyBool_FromLong,3.2,, data,PyBool_Type,3.2,, @@ -126,6 +160,7 @@ func,PyDict_Merge,3.2,, func,PyDict_MergeFromSeq2,3.2,, func,PyDict_New,3.2,, func,PyDict_Next,3.2,, +func,PyDict_SetDefaultRef,3.15,, func,PyDict_SetItem,3.2,, func,PyDict_SetItemString,3.2,, func,PyDict_Size,3.2,, @@ -354,8 +389,14 @@ func,PyList_SetSlice,3.2,, func,PyList_Size,3.2,, func,PyList_Sort,3.2,, data,PyList_Type,3.2,, +type,PyLongExport,3.15,,full-abi +type,PyLongLayout,3.15,,full-abi type,PyLongObject,3.2,,opaque data,PyLongRangeIter_Type,3.2,, +type,PyLongWriter,3.15,,opaque +func,PyLongWriter_Create,3.15,, +func,PyLongWriter_Discard,3.15,, +func,PyLongWriter_Finish,3.15,, func,PyLong_AsDouble,3.2,, func,PyLong_AsInt,3.13,, func,PyLong_AsInt32,3.14,, @@ -374,6 +415,8 @@ func,PyLong_AsUnsignedLongLong,3.2,, func,PyLong_AsUnsignedLongLongMask,3.2,, func,PyLong_AsUnsignedLongMask,3.2,, func,PyLong_AsVoidPtr,3.2,, +func,PyLong_Export,3.15,, +func,PyLong_FreeExport,3.15,, func,PyLong_FromDouble,3.2,, func,PyLong_FromInt32,3.14,, func,PyLong_FromInt64,3.14,, @@ -390,7 +433,9 @@ func,PyLong_FromUnsignedLongLong,3.2,, func,PyLong_FromUnsignedNativeBytes,3.14,, func,PyLong_FromVoidPtr,3.2,, func,PyLong_GetInfo,3.2,, +func,PyLong_GetNativeLayout,3.15,, data,PyLong_Type,3.2,, +macro,PyMODEXPORT_FUNC,3.15,, data,PyMap_Type,3.2,, func,PyMapping_Check,3.2,, func,PyMapping_GetItemString,3.2,, @@ -425,9 +470,10 @@ func,PyMemoryView_GetContiguous,3.2,, data,PyMemoryView_Type,3.2,, type,PyMethodDef,3.2,,full-abi data,PyMethodDescr_Type,3.2,, -type,PyModuleDef,3.2,,full-abi -type,PyModuleDef_Base,3.2,,full-abi +type,PyModuleDef,3.2,,abi3t-opaque +type,PyModuleDef_Base,3.2,,abi3t-opaque func,PyModuleDef_Init,3.5,, +type,PyModuleDef_Slot,3.5,,full-abi data,PyModuleDef_Type,3.5,, func,PyModule_Add,3.13,, func,PyModule_AddFunctions,3.7,, @@ -437,8 +483,10 @@ func,PyModule_AddObjectRef,3.10,, func,PyModule_AddStringConstant,3.2,, func,PyModule_AddType,3.10,, func,PyModule_Create2,3.2,, +func,PyModule_Exec,3.15,, func,PyModule_ExecDef,3.7,, func,PyModule_FromDefAndSpec2,3.7,, +func,PyModule_FromSlotsAndSpec,3.15,, func,PyModule_GetDef,3.2,, func,PyModule_GetDict,3.2,, func,PyModule_GetFilename,3.2,, @@ -446,6 +494,10 @@ func,PyModule_GetFilenameObject,3.2,, func,PyModule_GetName,3.2,, func,PyModule_GetNameObject,3.7,, func,PyModule_GetState,3.2,, +func,PyModule_GetStateSize,3.15,, +func,PyModule_GetState_DuringGC,3.15,, +func,PyModule_GetToken,3.15,, +func,PyModule_GetToken_DuringGC,3.15,, func,PyModule_New,3.2,, func,PyModule_NewObject,3.7,, func,PyModule_SetDocString,3.7,, @@ -548,6 +600,7 @@ func,PyObject_GetIter,3.2,, func,PyObject_GetOptionalAttr,3.13,, func,PyObject_GetOptionalAttrString,3.13,, func,PyObject_GetTypeData,3.12,, +func,PyObject_GetTypeData_DuringGC,3.15,, func,PyObject_HasAttr,3.2,, func,PyObject_HasAttrString,3.2,, func,PyObject_HasAttrStringWithError,3.13,, @@ -637,7 +690,6 @@ func,PySys_GetObject,3.2,, func,PySys_GetOptionalAttr,3.15,, func,PySys_GetOptionalAttrString,3.15,, func,PySys_GetXOptions,3.7,, -func,PySys_ResetWarnOptions,3.2,, func,PySys_SetArgv,3.2,, func,PySys_SetArgvEx,3.2,, func,PySys_SetObject,3.2,, @@ -701,12 +753,17 @@ func,PyType_FromSpecWithBases,3.3,, func,PyType_GenericAlloc,3.2,, func,PyType_GenericNew,3.2,, func,PyType_GetBaseByToken,3.14,, +func,PyType_GetBaseByToken_DuringGC,3.15,, func,PyType_GetFlags,3.2,, func,PyType_GetFullyQualifiedName,3.13,, func,PyType_GetModule,3.10,, func,PyType_GetModuleByDef,3.13,, +func,PyType_GetModuleByToken,3.15,, +func,PyType_GetModuleByToken_DuringGC,3.15,, func,PyType_GetModuleName,3.13,, func,PyType_GetModuleState,3.10,, +func,PyType_GetModuleState_DuringGC,3.15,, +func,PyType_GetModule_DuringGC,3.15,, func,PyType_GetName,3.11,, func,PyType_GetQualName,3.11,, func,PyType_GetSlot,3.4,, @@ -831,13 +888,20 @@ member,PyVarObject.ob_size,3.2,, func,PyVectorcall_Call,3.12,, func,PyVectorcall_NARGS,3.12,, type,PyWeakReference,3.2,,opaque -func,PyWeakref_GetObject,3.2,, func,PyWeakref_GetRef,3.13,, func,PyWeakref_NewProxy,3.2,, func,PyWeakref_NewRef,3.2,, data,PyWrapperDescr_Type,3.2,, func,PyWrapper_New,3.2,, data,PyZip_Type,3.2,, +macro,Py_ASNATIVEBYTES_ALLOW_INDEX,3.14,, +macro,Py_ASNATIVEBYTES_BIG_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_DEFAULTS,3.14,, +macro,Py_ASNATIVEBYTES_LITTLE_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_NATIVE_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_REJECT_NEGATIVE,3.14,, +macro,Py_ASNATIVEBYTES_UNSIGNED_BUFFER,3.14,, +macro,Py_AUDIT_READ,3.12,, func,Py_AddPendingCall,3.2,, func,Py_AtExit,3.2,, macro,Py_BEGIN_ALLOW_THREADS,3.2,, @@ -868,6 +932,7 @@ func,Py_GetPlatform,3.2,, func,Py_GetRecursionLimit,3.2,, func,Py_GetVersion,3.2,, data,Py_HasFileSystemDefaultEncoding,3.2,, +func,Py_IS_TYPE,3.15,, func,Py_IncRef,3.2,, func,Py_Initialize,3.2,, func,Py_InitializeEx,3.2,, @@ -884,22 +949,147 @@ func,Py_NewInterpreter,3.2,, func,Py_NewRef,3.10,, func,Py_PACK_FULL_VERSION,3.14,, func,Py_PACK_VERSION,3.14,, +macro,Py_READONLY,3.12,, func,Py_REFCNT,3.14,, +macro,Py_RELATIVE_OFFSET,3.12,, func,Py_ReprEnter,3.2,, func,Py_ReprLeave,3.2,, +func,Py_SET_SIZE,3.15,, +func,Py_SIZE,3.15,, func,Py_SetProgramName,3.2,, func,Py_SetPythonHome,3.2,, func,Py_SetRecursionLimit,3.2,, +macro,Py_TPFLAGS_BASETYPE,3.2,, +macro,Py_TPFLAGS_DEFAULT,3.2,, +macro,Py_TPFLAGS_HAVE_GC,3.2,, +macro,Py_TPFLAGS_HAVE_VECTORCALL,3.12,, +macro,Py_TPFLAGS_ITEMS_AT_END,3.12,, +macro,Py_TPFLAGS_METHOD_DESCRIPTOR,3.8,, +macro,Py_TP_USE_SPEC,3.14,, func,Py_TYPE,3.14,, +macro,Py_T_BOOL,3.12,, +macro,Py_T_BYTE,3.12,, +macro,Py_T_CHAR,3.12,, +macro,Py_T_DOUBLE,3.12,, +macro,Py_T_FLOAT,3.12,, +macro,Py_T_INT,3.12,, +macro,Py_T_LONG,3.12,, +macro,Py_T_LONGLONG,3.12,, +macro,Py_T_OBJECT_EX,3.12,, +macro,Py_T_PYSSIZET,3.12,, +macro,Py_T_SHORT,3.12,, +macro,Py_T_STRING,3.12,, +macro,Py_T_STRING_INPLACE,3.12,, +macro,Py_T_UBYTE,3.12,, +macro,Py_T_UINT,3.12,, +macro,Py_T_ULONG,3.12,, +macro,Py_T_ULONGLONG,3.12,, +macro,Py_T_USHORT,3.12,, type,Py_UCS4,3.2,, macro,Py_UNBLOCK_THREADS,3.2,, data,Py_UTF8Mode,3.8,, func,Py_VaBuildValue,3.2,, data,Py_Version,3.11,, func,Py_XNewRef,3.10,, +macro,Py_am_aiter,3.5,, +macro,Py_am_anext,3.5,, +macro,Py_am_await,3.5,, +macro,Py_am_send,3.10,, +macro,Py_bf_getbuffer,3.11,, +macro,Py_bf_releasebuffer,3.11,, type,Py_buffer,3.11,,full-abi type,Py_intptr_t,3.2,, +macro,Py_mod_abi,3.15,, +macro,Py_mod_create,3.5,, +macro,Py_mod_doc,3.15,, +macro,Py_mod_exec,3.5,, +macro,Py_mod_gil,3.13,, +macro,Py_mod_methods,3.15,, +macro,Py_mod_multiple_interpreters,3.12,, +macro,Py_mod_name,3.15,, +macro,Py_mod_state_clear,3.15,, +macro,Py_mod_state_free,3.15,, +macro,Py_mod_state_size,3.15,, +macro,Py_mod_state_traverse,3.15,, +macro,Py_mod_token,3.15,, +macro,Py_mp_ass_subscript,3.2,, +macro,Py_mp_length,3.2,, +macro,Py_mp_subscript,3.2,, +macro,Py_nb_absolute,3.2,, +macro,Py_nb_add,3.2,, +macro,Py_nb_and,3.2,, +macro,Py_nb_bool,3.2,, +macro,Py_nb_divmod,3.2,, +macro,Py_nb_float,3.2,, +macro,Py_nb_floor_divide,3.2,, +macro,Py_nb_index,3.2,, +macro,Py_nb_inplace_add,3.2,, +macro,Py_nb_inplace_and,3.2,, +macro,Py_nb_inplace_floor_divide,3.2,, +macro,Py_nb_inplace_lshift,3.2,, +macro,Py_nb_inplace_matrix_multiply,3.5,, +macro,Py_nb_inplace_multiply,3.2,, +macro,Py_nb_inplace_or,3.2,, +macro,Py_nb_inplace_power,3.2,, +macro,Py_nb_inplace_remainder,3.2,, +macro,Py_nb_inplace_rshift,3.2,, +macro,Py_nb_inplace_subtract,3.2,, +macro,Py_nb_inplace_true_divide,3.2,, +macro,Py_nb_inplace_xor,3.2,, +macro,Py_nb_int,3.2,, +macro,Py_nb_invert,3.2,, +macro,Py_nb_lshift,3.2,, +macro,Py_nb_matrix_multiply,3.5,, +macro,Py_nb_multiply,3.2,, +macro,Py_nb_negative,3.2,, +macro,Py_nb_or,3.2,, +macro,Py_nb_positive,3.2,, +macro,Py_nb_power,3.2,, +macro,Py_nb_remainder,3.2,, +macro,Py_nb_rshift,3.2,, +macro,Py_nb_subtract,3.2,, +macro,Py_nb_true_divide,3.2,, +macro,Py_nb_xor,3.2,, +macro,Py_sq_ass_item,3.2,, +macro,Py_sq_concat,3.2,, +macro,Py_sq_contains,3.2,, +macro,Py_sq_inplace_concat,3.2,, +macro,Py_sq_inplace_repeat,3.2,, +macro,Py_sq_item,3.2,, +macro,Py_sq_length,3.2,, +macro,Py_sq_repeat,3.2,, type,Py_ssize_t,3.2,, +macro,Py_tp_alloc,3.2,, +macro,Py_tp_base,3.2,, +macro,Py_tp_bases,3.2,, +macro,Py_tp_call,3.2,, +macro,Py_tp_clear,3.2,, +macro,Py_tp_dealloc,3.2,, +macro,Py_tp_del,3.2,, +macro,Py_tp_descr_get,3.2,, +macro,Py_tp_descr_set,3.2,, +macro,Py_tp_doc,3.2,, +macro,Py_tp_finalize,3.5,, +macro,Py_tp_free,3.2,, +macro,Py_tp_getattr,3.2,, +macro,Py_tp_getattro,3.2,, +macro,Py_tp_getset,3.2,, +macro,Py_tp_hash,3.2,, +macro,Py_tp_init,3.2,, +macro,Py_tp_is_gc,3.2,, +macro,Py_tp_iter,3.2,, +macro,Py_tp_iternext,3.2,, +macro,Py_tp_members,3.2,, +macro,Py_tp_methods,3.2,, +macro,Py_tp_new,3.2,, +macro,Py_tp_repr,3.2,, +macro,Py_tp_richcompare,3.2,, +macro,Py_tp_setattr,3.2,, +macro,Py_tp_setattro,3.2,, +macro,Py_tp_str,3.2,, +macro,Py_tp_token,3.14,, +macro,Py_tp_traverse,3.2,, +macro,Py_tp_vectorcall,3.14,, type,Py_uintptr_t,3.2,, type,allocfunc,3.2,, type,binaryfunc,3.2,, diff --git a/Doc/data/threadsafety.dat b/Doc/data/threadsafety.dat new file mode 100644 index 00000000000..ea5a24a5505 --- /dev/null +++ b/Doc/data/threadsafety.dat @@ -0,0 +1,284 @@ +# Thread safety annotations for C API functions. +# +# Each line has the form: +# function_name : level +# +# Where level is one of: +# incompatible -- not safe even with external locking +# compatible -- safe if the caller serializes all access with external locks +# distinct -- safe on distinct objects without external synchronization +# shared -- safe for concurrent use on the same object +# atomic -- atomic +# +# Lines beginning with '#' are ignored. +# The function name must match the C domain identifier used in the documentation. + +# Synchronization primitives (Doc/c-api/synchronization.rst) +PyMutex_Lock:atomic: +PyMutex_Unlock:atomic: +PyMutex_IsLocked:atomic: + + +# Dictionary objects (Doc/c-api/dict.rst) + +# Type checks - read ob_type pointer, always safe +PyDict_Check:atomic: +PyDict_CheckExact:atomic: + +# Creation - pure allocation, no shared state +PyDict_New:atomic: + +# Lock-free lookups - use _Py_dict_lookup_threadsafe(), no locking. +# Atomic with simple types. +PyDict_Contains:shared: +PyDict_ContainsString:atomic: +PyDict_GetItemRef:shared: +PyDict_GetItemStringRef:atomic: +PyDict_Size:atomic: +PyDict_GET_SIZE:atomic: + +# Borrowed-reference lookups - lock-free dict access but returned +# borrowed reference is unsafe in free-threaded builds without +# external synchronization +PyDict_GetItem:compatible: +PyDict_GetItemWithError:compatible: +PyDict_GetItemString:compatible: +PyDict_SetDefault:compatible: + +# Iteration - no locking; returns borrowed refs +PyDict_Next:compatible: + +# Single-item mutations - protected by per-object critical section +PyDict_SetItem:shared: +PyDict_SetItemString:atomic: +PyDict_DelItem:shared: +PyDict_DelItemString:atomic: +PyDict_SetDefaultRef:shared: +PyDict_Pop:shared: +PyDict_PopString:atomic: + +# Bulk reads - hold per-object lock for duration +PyDict_Clear:atomic: +PyDict_Copy:atomic: +PyDict_Keys:atomic: +PyDict_Values:atomic: +PyDict_Items:atomic: + +# Merge/update - lock target dict; also lock source when it is a dict +PyDict_Update:shared: +PyDict_Merge:shared: +PyDict_MergeFromSeq2:shared: + +# Watcher registration - no synchronization on interpreter state +PyDict_AddWatcher:compatible: +PyDict_ClearWatcher:compatible: + +# Per-dict watcher tags - non-atomic RMW on _ma_watcher_tag; +# safe on distinct dicts only +PyDict_Watch:distinct: +PyDict_Unwatch:distinct: + + +# List objects (Doc/c-api/list.rst) + +# Type checks - read ob_type pointer, always safe +PyList_Check:atomic: +PyList_CheckExact:atomic: + +# Creation - pure allocation, no shared state +PyList_New:atomic: + +# Size - uses atomic load on free-threaded builds +PyList_Size:atomic: +PyList_GET_SIZE:atomic: + +# Strong-reference lookup - lock-free with atomic ops +PyList_GetItemRef:atomic: + +# Borrowed-reference lookups - no locking; returned borrowed +# reference is unsafe in free-threaded builds without +# external synchronization +PyList_GetItem:compatible: +PyList_GET_ITEM:compatible: + +# Single-item mutations - hold per-object lock for duration; +# appear atomic to lock-free readers +PyList_SetItem:atomic: +PyList_Append:atomic: + +# Insert - protected by per-object critical section; shifts +# elements so lock-free readers may observe intermediate states +PyList_Insert:shared: + +# Initialization macro - no synchronization; normally only used +# to fill in new lists where there is no previous content +PyList_SET_ITEM:compatible: + +# Bulk operations - hold per-object lock for duration +PyList_GetSlice:atomic: +PyList_AsTuple:atomic: +PyList_Clear:atomic: + +# Reverse - protected by per-object critical section; swaps +# elements so lock-free readers may observe intermediate states +PyList_Reverse:shared: + +# Slice assignment - lock target list; also lock source when it +# is a list +PyList_SetSlice:shared: + +# Sort - per-object lock held; the list is emptied before sorting +# so other threads may observe an empty list, but they won't see the +# intermediate states of the sort +PyList_Sort:shared: + +# Extend - lock target list; also lock source when it is a +# list, set, or dict +PyList_Extend:shared: + +# Creation - pure allocation, no shared state +PyBytes_FromString:atomic: +PyBytes_FromStringAndSize:atomic: +PyBytes_DecodeEscape:atomic: + +# Creation from formatting C primitives - pure allocation, no shared state +PyBytes_FromFormat:atomic: +PyBytes_FromFormatV:atomic: + +# Creation from object - uses buffer protocol so may call arbitrary code; +# safe as long as the buffer is not mutated by another thread during the operation +PyBytes_FromObject:shared: + +# Size - uses atomic load on free-threaded builds +PyBytes_Size:atomic: +PyBytes_GET_SIZE:atomic: + +# Raw data - no locking; mutating it is unsafe if the bytes object is shared between threads +PyBytes_AsString:compatible: +PyBytes_AS_STRING:compatible: +PyBytes_AsStringAndSize:compatible: + +# Concatenation - uses buffer protocol; safe as long as buffer is not mutated by another thread during the operation +PyBytes_Concat:shared: +PyBytes_ConcatAndDel:shared: +PyBytes_Join:shared: + +# Resizing - safe if the object is unique +_PyBytes_Resize:distinct: + +# Repr - atomic as bytes are immutable +PyBytes_Repr:atomic: + +# Creation from object - may call arbitrary code +PyByteArray_FromObject:shared: + +# Creation - pure allocation, no shared state +PyByteArray_FromStringAndSize:atomic: + +# Concatenation - uses buffer protocol; safe as long as buffer is not mutated by another thread during the operation +PyByteArray_Concat:shared: + +# Size - uses atomic load on free-threaded builds +PyByteArray_Size:atomic: +PyByteArray_GET_SIZE:atomic: + +# Raw data - no locking; mutating it is unsafe if the bytearray object is shared between threads +PyByteArray_AsString:compatible: +PyByteArray_AS_STRING:compatible: + +# Creation - may iterate the iterable argument, calling arbitrary code. +# Atomic for sets, frozensets, dicts, and frozendicts. +PySet_New:shared: +PyFrozenSet_New:shared: + +# Size - uses atomic load on free-threaded builds +PySet_Size:atomic: +PySet_GET_SIZE:atomic: + +# Contains - lock-free, atomic with simple types +PySet_Contains:shared: + +# Mutations - hold per-object lock for duration +# atomic with simple types +PySet_Add:shared: +PySet_Discard:shared: + +# Pop - hold per-object lock for duration +PySet_Pop:atomic: + +# Clear - empties the set before clearing +PySet_Clear:atomic: + +# Capsule objects (Doc/c-api/capsule.rst) + +# Type check - read ob_type pointer, always safe +PyCapsule_CheckExact:atomic: + +# Creation - pure allocation, no shared state +PyCapsule_New:atomic: + +# Validation - reads pointer and name fields; safe on distinct objects +PyCapsule_IsValid:distinct: + +# Getters - read struct fields; safe on distinct objects but +# concurrent access to the same capsule requires external synchronization +PyCapsule_GetPointer:distinct: +PyCapsule_GetName:distinct: +PyCapsule_GetDestructor:distinct: +PyCapsule_GetContext:distinct: + +# Setters - write struct fields; safe on distinct objects but +# concurrent access to the same capsule requires external synchronization +PyCapsule_SetPointer:distinct: +PyCapsule_SetName:distinct: +PyCapsule_SetDestructor:distinct: +PyCapsule_SetContext:distinct: + +# Import - looks up a capsule from a module attribute and +# calls PyCapsule_GetPointer; may call arbitrary code +PyCapsule_Import:compatible: + +# Tuple objects + +# Creation - pure allocation, no shared state +PyTuple_New:atomic: +PyTuple_FromArray:atomic: +PyTuple_Pack:atomic: + +# Size - tuples are immutable so size never changes +PyTuple_Size:atomic: +PyTuple_GET_SIZE:atomic: + +# Borrowed-reference lookups - tuples are immutable so items +# never change, however the tuple must be kept alive while using the borrowed reference +PyTuple_GetItem:compatible: +PyTuple_GET_ITEM:compatible: + +# Slice - creates a new tuple from an existing tuple +PyTuple_GetSlice:atomic: + +# SetItem - only usable on tuples with refcount 1 +PyTuple_SetItem:compatible: +PyTuple_SET_ITEM:compatible: + +# Resize - only usable on tuples with refcount 1 +_PyTuple_Resize:compatible: + +# Struct Sequence objects + +# Creation +PyStructSequence_NewType:atomic: +PyStructSequence_New:atomic: + +# Initialization - modifies the type object in place +PyStructSequence_InitType:distinct: +PyStructSequence_InitType2:distinct: + +# Borrowed-reference lookups - same as tuple items +PyStructSequence_GetItem:compatible: +PyStructSequence_GET_ITEM:compatible: + +# SetItem - only for filling in brand new instances +PyStructSequence_SetItem:compatible: +PyStructSequence_SET_ITEM:compatible: + diff --git a/Doc/deprecations/c-api-pending-removal-in-3.15.rst b/Doc/deprecations/c-api-pending-removal-in-3.15.rst index a3e335ecaf4..789ec83d2d9 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.15.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.15.rst @@ -3,12 +3,10 @@ Pending removal in Python 3.15 * The :c:func:`!PyImport_ImportModuleNoBlock`: Use :c:func:`PyImport_ImportModule` instead. -* :c:func:`PyWeakref_GetObject` and :c:func:`PyWeakref_GET_OBJECT`: +* :c:func:`!PyWeakref_GetObject` and :c:func:`!PyWeakref_GET_OBJECT`: Use :c:func:`PyWeakref_GetRef` instead. The `pythoncapi-compat project `__ can be used to get :c:func:`PyWeakref_GetRef` on Python 3.12 and older. -* :c:type:`Py_UNICODE` type and the :c:macro:`!Py_UNICODE_WIDE` macro: - Use :c:type:`wchar_t` instead. * :c:func:`!PyUnicode_AsDecodedObject`: Use :c:func:`PyCodec_Decode` instead. * :c:func:`!PyUnicode_AsDecodedUnicode`: @@ -59,7 +57,7 @@ Pending removal in Python 3.15 Set :c:member:`PyConfig.program_name` instead. * :c:func:`!Py_SetPythonHome()`: Set :c:member:`PyConfig.home` instead. - * :c:func:`PySys_ResetWarnOptions`: + * :c:func:`!PySys_ResetWarnOptions`: Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. The :c:func:`Py_InitializeFromConfig` API should be used with diff --git a/Doc/deprecations/c-api-pending-removal-in-3.18.rst b/Doc/deprecations/c-api-pending-removal-in-3.18.rst index 564cfb79a0b..022aee93aa7 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.18.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.18.rst @@ -1,11 +1,12 @@ Pending removal in Python 3.18 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* Deprecated private functions (:gh:`128863`): +* The following private functions are deprecated + and planned for removal in Python 3.18: * :c:func:`!_PyBytes_Join`: use :c:func:`PyBytes_Join`. * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`. - * :c:func:`!_PyDict_Pop()`: :c:func:`PyDict_Pop`. + * :c:func:`!_PyDict_Pop()`: use :c:func:`PyDict_Pop`. * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`. * :c:func:`!_PyLong_FromDigits` and :c:func:`!_PyLong_New`: use :c:func:`PyLongWriter_Create`. @@ -31,7 +32,7 @@ Pending removal in Python 3.18 :c:func:`PyUnicodeWriter_WriteSubstring(writer, str, start, end) `. * :c:func:`!_PyUnicodeWriter_WriteASCIIString`: replace ``_PyUnicodeWriter_WriteASCIIString(&writer, str)`` with - :c:func:`PyUnicodeWriter_WriteUTF8(writer, str) `. + :c:func:`PyUnicodeWriter_WriteASCII(writer, str) `. * :c:func:`!_PyUnicodeWriter_WriteLatin1String`: replace ``_PyUnicodeWriter_WriteLatin1String(&writer, str)`` with :c:func:`PyUnicodeWriter_WriteUTF8(writer, str) `. @@ -41,5 +42,6 @@ Pending removal in Python 3.18 * :c:func:`!_Py_fopen_obj`: use :c:func:`Py_fopen`. The `pythoncapi-compat project - `__ can be used to get these - new public functions on Python 3.13 and older. + `__ can be used to get + these new public functions on Python 3.13 and older. + (Contributed by Victor Stinner in :gh:`128863`.) diff --git a/Doc/deprecations/c-api-pending-removal-in-3.19.rst b/Doc/deprecations/c-api-pending-removal-in-3.19.rst new file mode 100644 index 00000000000..ac9dcb8b424 --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.19.rst @@ -0,0 +1,4 @@ +Pending removal in Python 3.19 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* :pep:`456` embedders support for the string hashing scheme definition. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.20.rst b/Doc/deprecations/c-api-pending-removal-in-3.20.rst index 82f975d6ed4..8de55bbe7e6 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.20.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.20.rst @@ -1,7 +1,16 @@ Pending removal in Python 3.20 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +* :c:func:`!_PyObject_CallMethodId`, :c:func:`!_PyObject_GetAttrId` and + :c:func:`!_PyUnicode_FromId` are deprecated since 3.15 and will be removed in + 3.20. Instead, use :c:func:`PyUnicode_InternFromString()` and cache the result in + the module state, then call :c:func:`PyObject_CallMethod` or + :c:func:`PyObject_GetAttr`. + (Contributed by Victor Stinner in :gh:`141049`.) + * The ``cval`` field in :c:type:`PyComplexObject` (:gh:`128813`). Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex` to convert a Python complex number to/from the C :c:type:`Py_complex` representation. + +* Macros :c:macro:`!Py_MATH_PIl` and :c:macro:`!Py_MATH_El`. diff --git a/Doc/deprecations/index.rst b/Doc/deprecations/index.rst index c6e05c176b2..eedcd2e9c9d 100644 --- a/Doc/deprecations/index.rst +++ b/Doc/deprecations/index.rst @@ -7,17 +7,27 @@ Deprecations .. include:: pending-removal-in-3.17.rst +.. include:: pending-removal-in-3.18.rst + .. include:: pending-removal-in-3.19.rst +.. include:: pending-removal-in-3.20.rst + .. include:: pending-removal-in-future.rst +.. include:: soft-deprecations.rst + C API deprecations ------------------ .. include:: c-api-pending-removal-in-3.15.rst +.. include:: c-api-pending-removal-in-3.16.rst + .. include:: c-api-pending-removal-in-3.18.rst +.. include:: c-api-pending-removal-in-3.19.rst + .. include:: c-api-pending-removal-in-3.20.rst .. include:: c-api-pending-removal-in-future.rst diff --git a/Doc/deprecations/pending-removal-in-3.13.rst b/Doc/deprecations/pending-removal-in-3.13.rst index 2fd2f12cc6a..d5b8c80e8f9 100644 --- a/Doc/deprecations/pending-removal-in-3.13.rst +++ b/Doc/deprecations/pending-removal-in-3.13.rst @@ -38,15 +38,3 @@ APIs: * :meth:`!unittest.TestProgram.usageExit` (:gh:`67048`) * :class:`!webbrowser.MacOSX` (:gh:`86421`) * :class:`classmethod` descriptor chaining (:gh:`89519`) -* :mod:`importlib.resources` deprecated methods: - - * ``contents()`` - * ``is_resource()`` - * ``open_binary()`` - * ``open_text()`` - * ``path()`` - * ``read_binary()`` - * ``read_text()`` - - Use :func:`importlib.resources.files` instead. Refer to `importlib-resources: Migrating from Legacy - `_ (:gh:`106531`) diff --git a/Doc/deprecations/pending-removal-in-3.14.rst b/Doc/deprecations/pending-removal-in-3.14.rst index 9aac10840a6..171758156ab 100644 --- a/Doc/deprecations/pending-removal-in-3.14.rst +++ b/Doc/deprecations/pending-removal-in-3.14.rst @@ -38,12 +38,6 @@ Pending removal in Python 3.14 is no current event loop set and it decides to create one. (Contributed by Serhiy Storchaka and Guido van Rossum in :gh:`100160`.) -* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. - Prefer :class:`!Sequence` or :class:`~collections.abc.Buffer`. - For use in typing, prefer a union, like ``bytes | bytearray``, - or :class:`collections.abc.Buffer`. - (Contributed by Shantanu Jain in :gh:`91896`.) - * :mod:`email`: Deprecated the *isdst* parameter in :func:`email.utils.localtime`. (Contributed by Alan Williams in :gh:`72346`.) @@ -96,9 +90,6 @@ Pending removal in Python 3.14 if :ref:`named placeholders ` are used and *parameters* is a sequence instead of a :class:`dict`. -* :mod:`typing`: :class:`!typing.ByteString`, deprecated since Python 3.9, - now causes a :exc:`DeprecationWarning` to be emitted when it is used. - * :mod:`urllib`: :class:`!urllib.parse.Quoter` is deprecated: it was not intended to be a public API. diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst index 076346be621..1d9a3095813 100644 --- a/Doc/deprecations/pending-removal-in-3.15.rst +++ b/Doc/deprecations/pending-removal-in-3.15.rst @@ -3,9 +3,9 @@ Pending removal in Python 3.15 * The import system: - * Setting :attr:`~module.__cached__` on a module while + * Setting ``__cached__`` on a module while failing to set :attr:`__spec__.cached ` - is deprecated. In Python 3.15, :attr:`!__cached__` will cease to be set or + is deprecated. In Python 3.15, ``__cached__`` will cease to be set or take into consideration by the import system or standard library. (:gh:`97879`) * Setting :attr:`~module.__package__` on a module while @@ -33,16 +33,6 @@ Pending removal in Python 3.15 * ``load_module()`` method: use ``exec_module()`` instead. -* :class:`locale`: - - * The :func:`~locale.getdefaultlocale` function - has been deprecated since Python 3.11. - Its removal was originally planned for Python 3.13 (:gh:`90817`), - but has been postponed to Python 3.15. - Use :func:`~locale.getlocale`, :func:`~locale.setlocale`, - and :func:`~locale.getencoding` instead. - (Contributed by Hugo van Kemenade in :gh:`111187`.) - * :mod:`pathlib`: * :meth:`!.PurePath.is_reserved` @@ -64,13 +54,13 @@ Pending removal in Python 3.15 * :func:`~threading.RLock` will take no arguments in Python 3.15. Passing any arguments has been deprecated since Python 3.14, - as the Python version does not permit any arguments, + as the Python version does not permit any arguments, but the C version allows any number of positional or keyword arguments, ignoring every argument. * :mod:`types`: - * :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was + * :class:`types.CodeType`: Accessing :attr:`!codeobject.co_lnotab` was deprecated in :pep:`626` since 3.10 and was planned to be removed in 3.12, but it only got a proper :exc:`DeprecationWarning` in 3.12. @@ -92,7 +82,7 @@ Pending removal in Python 3.15 Use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})`` to create a TypedDict with zero field. - * The :func:`typing.no_type_check_decorator` decorator function + * The :func:`!typing.no_type_check_decorator` decorator function has been deprecated since Python 3.13. After eight years in the :mod:`typing` module, it has yet to be supported by any major type checker. diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst index cdd76ee693f..b00c7002b03 100644 --- a/Doc/deprecations/pending-removal-in-3.16.rst +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -63,9 +63,9 @@ Pending removal in Python 3.16 * :mod:`logging`: - Support for custom logging handlers with the *strm* argument is deprecated - and scheduled for removal in Python 3.16. Define handlers with the *stream* - argument instead. (Contributed by Mariusz Felisiak in :gh:`115032`.) + * Support for custom logging handlers with the *strm* argument is deprecated + and scheduled for removal in Python 3.16. Define handlers with the *stream* + argument instead. (Contributed by Mariusz Felisiak in :gh:`115032`.) * :mod:`mimetypes`: diff --git a/Doc/deprecations/pending-removal-in-3.17.rst b/Doc/deprecations/pending-removal-in-3.17.rst index 370b98307e5..952ffad6435 100644 --- a/Doc/deprecations/pending-removal-in-3.17.rst +++ b/Doc/deprecations/pending-removal-in-3.17.rst @@ -1,6 +1,42 @@ Pending removal in Python 3.17 ------------------------------ +* :mod:`datetime`: + + * :meth:`~datetime.datetime.strptime` calls using a format string containing + ``%e`` (day of month) without a year. + This has been deprecated since Python 3.15. + (Contributed by Stan Ulbrych in :gh:`70647`.) + + +* :mod:`collections.abc`: + + - :class:`collections.abc.ByteString` is scheduled for removal in Python 3.17. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the :ref:`buffer protocol ` at runtime. For use + in type annotations, either use :class:`~collections.abc.Buffer` or a union + that explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was an + instance of :class:`!ByteString` never actually told you anything useful + about the object. Other common buffer types such as :class:`memoryview` + were also never understood as subtypes of :class:`!ByteString` (either at + runtime or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. + (Contributed by Shantanu Jain in :gh:`91896`.) + + +* :mod:`encodings`: + + - Passing non-ascii *encoding* names to :func:`encodings.normalize_encoding` + is deprecated and scheduled for removal in Python 3.17. + (Contributed by Stan Ulbrych in :gh:`136702`.) + * :mod:`typing`: - Before Python 3.14, old-style unions were implemented using the private class @@ -8,3 +44,22 @@ Pending removal in Python 3.17 but it has been retained for backward compatibility, with removal scheduled for Python 3.17. Users should use documented introspection helpers like :func:`typing.get_origin` and :func:`typing.get_args` instead of relying on private implementation details. + - :class:`typing.ByteString`, deprecated since Python 3.9, is scheduled for removal in + Python 3.17. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the :ref:`buffer protocol ` at runtime. For use + in type annotations, either use :class:`~collections.abc.Buffer` or a union + that explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was an + instance of :class:`!ByteString` never actually told you anything useful + about the object. Other common buffer types such as :class:`memoryview` + were also never understood as subtypes of :class:`!ByteString` (either at + runtime or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. + (Contributed by Shantanu Jain in :gh:`91896`.) diff --git a/Doc/deprecations/pending-removal-in-3.18.rst b/Doc/deprecations/pending-removal-in-3.18.rst new file mode 100644 index 00000000000..eb42fe9919e --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.18.rst @@ -0,0 +1,12 @@ +Pending removal in Python 3.18 +------------------------------ + +* No longer accept a boolean value when a file descriptor is expected. + (Contributed by Serhiy Storchaka in :gh:`82626`.) + +* :mod:`decimal`: + + * The non-standard and undocumented :class:`~decimal.Decimal` format + specifier ``'N'``, which is only supported in the :mod:`!decimal` module's + C implementation, has been deprecated since Python 3.13. + (Contributed by Serhiy Storchaka in :gh:`89902`.) diff --git a/Doc/deprecations/pending-removal-in-3.20.rst b/Doc/deprecations/pending-removal-in-3.20.rst new file mode 100644 index 00000000000..12d7acf5ce0 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.20.rst @@ -0,0 +1,45 @@ +Pending removal in Python 3.20 +------------------------------ + +* Calling the ``__new__()`` method of :class:`struct.Struct` without the + *format* argument is deprecated and will be removed in Python 3.20. Calling + :meth:`~object.__init__` method on initialized :class:`~struct.Struct` + objects is deprecated and will be removed in Python 3.20. + + (Contributed by Sergey B Kirpichev and Serhiy Storchaka in :gh:`143715`.) + +* The ``__version__``, ``version`` and ``VERSION`` attributes have been + deprecated in these standard library modules and will be removed in + Python 3.20. Use :py:data:`sys.version_info` instead. + + - :mod:`argparse` + - :mod:`csv` + - :mod:`ctypes` + - :mod:`!ctypes.macholib` + - :mod:`decimal` (use :data:`decimal.SPEC_VERSION` instead) + - :mod:`http.server` + - :mod:`imaplib` + - :mod:`ipaddress` + - :mod:`json` + - :mod:`logging` (``__date__`` also deprecated) + - :mod:`optparse` + - :mod:`pickle` + - :mod:`platform` + - :mod:`re` + - :mod:`socketserver` + - :mod:`tabnanny` + - :mod:`tarfile` + - :mod:`tkinter.font` + - :mod:`tkinter.ttk` + - :mod:`wsgiref.simple_server` + - :mod:`xml.etree.ElementTree` + - :mod:`!xml.sax.expatreader` + - :mod:`xml.sax.handler` + - :mod:`zlib` + + (Contributed by Hugo van Kemenade and Stan Ulbrych in :gh:`76007`.) + +* :mod:`ast`: + + * Creating instances of abstract AST nodes (such as :class:`ast.AST` + or :class:`!ast.expr`) is deprecated and will raise an error in Python 3.20. diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst index edb672ed8ad..74f98d33a4b 100644 --- a/Doc/deprecations/pending-removal-in-future.rst +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -15,7 +15,6 @@ although there is currently no date scheduled for their removal. * :mod:`builtins`: - * ``bool(NotImplemented)``. * Generators: ``throw(type, exc, tb)`` and ``athrow(type, exc, tb)`` signature is deprecated: use ``throw(exc)`` and ``athrow(exc)`` instead, the single argument signature. @@ -36,7 +35,6 @@ although there is currently no date scheduled for their removal. * Support for ``__complex__()`` method returning a strict subclass of :class:`complex`: these methods will be required to return an instance of :class:`complex`. - * Delegation of ``int()`` to ``__trunc__()`` method. * Passing a complex number as the *real* or *imag* argument in the :func:`complex` constructor is now deprecated; it should only be passed as a single positional argument. @@ -49,7 +47,7 @@ although there is currently no date scheduled for their removal. * :mod:`codecs`: use :func:`open` instead of :func:`codecs.open`. (:gh:`133038`) -* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method +* :attr:`!codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method instead. * :mod:`datetime`: @@ -77,7 +75,15 @@ although there is currently no date scheduled for their removal. * :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use BytesIO and binary mode instead. -* :mod:`os`: Calling :func:`os.register_at_fork` in multi-threaded process. +* :mod:`os`: Calling :func:`os.register_at_fork` in a multi-threaded process. + +* :mod:`os.path`: :func:`os.path.commonprefix` is deprecated, use + :func:`os.path.commonpath` for path prefixes. The :func:`os.path.commonprefix` + function is being deprecated due to having a misleading name and module. + The function is not safe to use for path prefixes despite being included in a + module about path manipulation, meaning it is easy to accidentally + introduce path traversal vulnerabilities into Python programs by using this + function. * :class:`!pydoc.ErrorDuringImport`: A tuple value for *exc_info* parameter is deprecated, use an exception instance. diff --git a/Doc/deprecations/soft-deprecations.rst b/Doc/deprecations/soft-deprecations.rst new file mode 100644 index 00000000000..a270052788e --- /dev/null +++ b/Doc/deprecations/soft-deprecations.rst @@ -0,0 +1,21 @@ +Soft deprecations +----------------- + +There are no plans to remove :term:`soft deprecated` APIs. + +* :func:`re.match` and :meth:`re.Pattern.match` are now + :term:`soft deprecated` in favor of the new :func:`re.prefixmatch` and + :meth:`re.Pattern.prefixmatch` APIs, which have been added as alternate, + more explicit names. These are intended to be used to alleviate confusion + around what *match* means by following the Zen of Python's *"Explicit is + better than implicit"* mantra. Most other language regular expression + libraries use an API named *match* to mean what Python has always called + *search*. + + We **do not** plan to remove the older :func:`!match` name, as it has been + used in code for over 30 years. Code supporting older versions of Python + should continue to use :func:`!match`, while new code should prefer + :func:`!prefixmatch`. See :ref:`prefixmatch-vs-match`. + + (Contributed by Gregory P. Smith in :gh:`86519` and + Hugo van Kemenade in :gh:`148100`.) diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index dee92312169..d33cbd2813d 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -3,154 +3,20 @@ .. _extending-intro: -****************************** -Extending Python with C or C++ -****************************** +******************************** +Using the C API: Assorted topics +******************************** -It is quite easy to add new built-in modules to Python, if you know how to -program in C. Such :dfn:`extension modules` can do two things that can't be -done directly in Python: they can implement new built-in object types, and they -can call C library functions and system calls. - -To support extensions, the Python API (Application Programmers Interface) -defines a set of functions, macros and variables that provide access to most -aspects of the Python run-time system. The Python API is incorporated in a C -source file by including the header ``"Python.h"``. - -The compilation of an extension module depends on its intended use as well as on -your system setup; details are given in later chapters. - -.. note:: - - The C extension interface is specific to CPython, and extension modules do - not work on other Python implementations. In many cases, it is possible to - avoid writing C extensions and preserve portability to other implementations. - For example, if your use case is calling C library functions or system calls, - you should consider using the :mod:`ctypes` module or the `cffi - `_ library rather than writing - custom C code. - These modules let you write Python code to interface with C code and are more - portable between implementations of Python than writing and compiling a C - extension module. - - -.. _extending-simpleexample: - -A Simple Example -================ - -Let's create an extension module called ``spam`` (the favorite food of Monty -Python fans...) and let's say we want to create a Python interface to the C -library function :c:func:`system` [#]_. This function takes a null-terminated -character string as argument and returns an integer. We want this function to -be callable from Python as follows: - -.. code-block:: pycon - - >>> import spam - >>> status = spam.system("ls -l") - -Begin by creating a file :file:`spammodule.c`. (Historically, if a module is -called ``spam``, the C file containing its implementation is called -:file:`spammodule.c`; if the module name is very long, like ``spammify``, the -module name can be just :file:`spammify.c`.) - -The first two lines of our file can be:: - - #define PY_SSIZE_T_CLEAN - #include - -which pulls in the Python API (you can add a comment describing the purpose of -the module and a copyright notice if you like). - -.. note:: - - Since Python may define some pre-processor definitions which affect the standard - headers on some systems, you *must* include :file:`Python.h` before any standard - headers are included. - - ``#define PY_SSIZE_T_CLEAN`` was used to indicate that ``Py_ssize_t`` should be - used in some APIs instead of ``int``. - It is not necessary since Python 3.13, but we keep it here for backward compatibility. - See :ref:`arg-parsing-string-and-buffers` for a description of this macro. - -All user-visible symbols defined by :file:`Python.h` have a prefix of ``Py`` or -``PY``, except those defined in standard header files. - -.. tip:: - - For backward compatibility, :file:`Python.h` includes several standard header files. - C extensions should include the standard headers that they use, - and should not rely on these implicit includes. - If using the limited C API version 3.13 or newer, the implicit includes are: - - * ```` - * ```` (on Windows) - * ```` - * ```` - * ```` - * ```` - * ```` - * ```` (if present) - - If :c:macro:`Py_LIMITED_API` is not defined, or is set to version 3.12 or older, - the headers below are also included: - - * ```` - * ```` (on POSIX) - - If :c:macro:`Py_LIMITED_API` is not defined, or is set to version 3.10 or older, - the headers below are also included: - - * ```` - * ```` - * ```` - * ```` - -The next thing we add to our module file is the C function that will be called -when the Python expression ``spam.system(string)`` is evaluated (we'll see -shortly how it ends up being called):: - - static PyObject * - spam_system(PyObject *self, PyObject *args) - { - const char *command; - int sts; - - if (!PyArg_ParseTuple(args, "s", &command)) - return NULL; - sts = system(command); - return PyLong_FromLong(sts); - } - -There is a straightforward translation from the argument list in Python (for -example, the single expression ``"ls -l"``) to the arguments passed to the C -function. The C function always has two arguments, conventionally named *self* -and *args*. - -The *self* argument points to the module object for module-level functions; -for a method it would point to the object instance. - -The *args* argument will be a pointer to a Python tuple object containing the -arguments. Each item of the tuple corresponds to an argument in the call's -argument list. The arguments are Python objects --- in order to do anything -with them in our C function we have to convert them to C values. The function -:c:func:`PyArg_ParseTuple` in the Python API checks the argument types and -converts them to C values. It uses a template string to determine the required -types of the arguments as well as the types of the C variables into which to -store the converted values. More about this later. - -:c:func:`PyArg_ParseTuple` returns true (nonzero) if all arguments have the right -type and its components have been stored in the variables whose addresses are -passed. It returns false (zero) if an invalid argument list was passed. In the -latter case it also raises an appropriate exception so the calling function can -return ``NULL`` immediately (as we saw in the example). +The :ref:`tutorial ` walked you through +creating a C API extension module, but left many areas unexplained. +This document looks at several concepts that you'll need to learn +in order to write more complex extensions. .. _extending-errors: -Intermezzo: Errors and Exceptions -================================= +Errors and Exceptions +===================== An important convention throughout the Python interpreter is the following: when a function fails, it should set an exception condition and return an error value @@ -321,194 +187,14 @@ call to :c:func:`PyErr_SetString` as shown below:: } -.. _backtoexample: - -Back to the Example -=================== - -Going back to our example function, you should now be able to understand this -statement:: - - if (!PyArg_ParseTuple(args, "s", &command)) - return NULL; - -It returns ``NULL`` (the error indicator for functions returning object pointers) -if an error is detected in the argument list, relying on the exception set by -:c:func:`PyArg_ParseTuple`. Otherwise the string value of the argument has been -copied to the local variable :c:data:`!command`. This is a pointer assignment and -you are not supposed to modify the string to which it points (so in Standard C, -the variable :c:data:`!command` should properly be declared as ``const char -*command``). - -The next statement is a call to the Unix function :c:func:`system`, passing it -the string we just got from :c:func:`PyArg_ParseTuple`:: - - sts = system(command); - -Our :func:`!spam.system` function must return the value of :c:data:`!sts` as a -Python object. This is done using the function :c:func:`PyLong_FromLong`. :: - - return PyLong_FromLong(sts); - -In this case, it will return an integer object. (Yes, even integers are objects -on the heap in Python!) - -If you have a C function that returns no useful argument (a function returning -:c:expr:`void`), the corresponding Python function must return ``None``. You -need this idiom to do so (which is implemented by the :c:macro:`Py_RETURN_NONE` -macro):: - - Py_INCREF(Py_None); - return Py_None; - -:c:data:`Py_None` is the C name for the special Python object ``None``. It is a -genuine Python object rather than a ``NULL`` pointer, which means "error" in most -contexts, as we have seen. - - -.. _methodtable: - -The Module's Method Table and Initialization Function -===================================================== - -I promised to show how :c:func:`!spam_system` is called from Python programs. -First, we need to list its name and address in a "method table":: - - static PyMethodDef spam_methods[] = { - ... - {"system", spam_system, METH_VARARGS, - "Execute a shell command."}, - ... - {NULL, NULL, 0, NULL} /* Sentinel */ - }; - -Note the third entry (``METH_VARARGS``). This is a flag telling the interpreter -the calling convention to be used for the C function. It should normally always -be ``METH_VARARGS`` or ``METH_VARARGS | METH_KEYWORDS``; a value of ``0`` means -that an obsolete variant of :c:func:`PyArg_ParseTuple` is used. - -When using only ``METH_VARARGS``, the function should expect the Python-level -parameters to be passed in as a tuple acceptable for parsing via -:c:func:`PyArg_ParseTuple`; more information on this function is provided below. - -The :c:macro:`METH_KEYWORDS` bit may be set in the third field if keyword -arguments should be passed to the function. In this case, the C function should -accept a third ``PyObject *`` parameter which will be a dictionary of keywords. -Use :c:func:`PyArg_ParseTupleAndKeywords` to parse the arguments to such a -function. - -The method table must be referenced in the module definition structure:: - - static struct PyModuleDef spam_module = { - ... - .m_methods = spam_methods, - ... - }; - -This structure, in turn, must be passed to the interpreter in the module's -initialization function. The initialization function must be named -:c:func:`!PyInit_name`, where *name* is the name of the module, and should be the -only non-\ ``static`` item defined in the module file:: - - PyMODINIT_FUNC - PyInit_spam(void) - { - return PyModuleDef_Init(&spam_module); - } - -Note that :c:macro:`PyMODINIT_FUNC` declares the function as ``PyObject *`` return type, -declares any special linkage declarations required by the platform, and for C++ -declares the function as ``extern "C"``. - -:c:func:`!PyInit_spam` is called when each interpreter imports its module -:mod:`!spam` for the first time. (See below for comments about embedding Python.) -A pointer to the module definition must be returned via :c:func:`PyModuleDef_Init`, -so that the import machinery can create the module and store it in ``sys.modules``. - -When embedding Python, the :c:func:`!PyInit_spam` function is not called -automatically unless there's an entry in the :c:data:`!PyImport_Inittab` table. -To add the module to the initialization table, use :c:func:`PyImport_AppendInittab`, -optionally followed by an import of the module:: - - #define PY_SSIZE_T_CLEAN - #include - - int - main(int argc, char *argv[]) - { - PyStatus status; - PyConfig config; - PyConfig_InitPythonConfig(&config); - - /* Add a built-in module, before Py_Initialize */ - if (PyImport_AppendInittab("spam", PyInit_spam) == -1) { - fprintf(stderr, "Error: could not extend in-built modules table\n"); - exit(1); - } - - /* Pass argv[0] to the Python interpreter */ - status = PyConfig_SetBytesString(&config, &config.program_name, argv[0]); - if (PyStatus_Exception(status)) { - goto exception; - } - - /* Initialize the Python interpreter. Required. - If this step fails, it will be a fatal error. */ - status = Py_InitializeFromConfig(&config); - if (PyStatus_Exception(status)) { - goto exception; - } - PyConfig_Clear(&config); - - /* Optionally import the module; alternatively, - import can be deferred until the embedded script - imports it. */ - PyObject *pmodule = PyImport_ImportModule("spam"); - if (!pmodule) { - PyErr_Print(); - fprintf(stderr, "Error: could not import module 'spam'\n"); - } - - // ... use Python C API here ... - - return 0; - - exception: - PyConfig_Clear(&config); - Py_ExitStatusException(status); - } - -.. note:: - - If you declare a global variable or a local static one, the module may - experience unintended side-effects on re-initialisation, for example when - removing entries from ``sys.modules`` or importing compiled modules into - multiple interpreters within a process - (or following a :c:func:`fork` without an intervening :c:func:`exec`). - If module state is not yet fully :ref:`isolated `, - authors should consider marking the module as having no support for subinterpreters - (via :c:macro:`Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED`). - -A more substantial example module is included in the Python source distribution -as :file:`Modules/xxlimited.c`. This file may be used as a template or simply -read as an example. - - .. _compilation: -Compilation and Linkage -======================= +Embedding an extension +====================== -There are two more things to do before you can use your new extension: compiling -and linking it with the Python system. If you use dynamic loading, the details -may depend on the style of dynamic loading your system uses; see the chapters -about building extension modules (chapter :ref:`building`) and additional -information that pertains only to building on Windows (chapter -:ref:`building-on-windows`) for more information about this. - -If you can't use dynamic loading, or if you want to make your module a permanent +If you want to make your module a permanent part of the Python interpreter, you will have to change the configuration setup -and rebuild the interpreter. Luckily, this is very simple on Unix: just place +and rebuild the interpreter. On Unix, place your file (:file:`spammodule.c` for example) in the :file:`Modules/` directory of an unpacked source distribution, add a line to the file :file:`Modules/Setup.local` describing your file: @@ -536,7 +222,7 @@ on the line in the configuration file as well, for instance: Calling Python Functions from C =============================== -So far we have concentrated on making C functions callable from Python. The +The tutorial concentrated on making C functions callable from Python. The reverse is also useful: calling Python functions from C. This is especially the case for libraries that support so-called "callback" functions. If a C interface makes use of callbacks, the equivalent Python often needs to provide a @@ -581,7 +267,7 @@ be part of a module definition:: } This function must be registered with the interpreter using the -:c:macro:`METH_VARARGS` flag; this is described in section :ref:`methodtable`. The +:c:macro:`METH_VARARGS` flag in :c:type:`PyMethodDef.ml_flags`. The :c:func:`PyArg_ParseTuple` function and its arguments are documented in section :ref:`parsetuple`. @@ -676,14 +362,21 @@ the above example, we use :c:func:`Py_BuildValue` to construct the dictionary. : Py_DECREF(result); +.. index:: single: PyArg_ParseTuple (C function) + .. _parsetuple: Extracting Parameters in Extension Functions ============================================ -.. index:: single: PyArg_ParseTuple (C function) +The :ref:`tutorial ` uses a ":c:data:`METH_O`" +function, which is limited to a single Python argument. +If you want more, you can use :c:data:`METH_VARARGS` instead. +With this flag, the C function will receive a :py:class:`tuple` of arguments +instead of a single object. -The :c:func:`PyArg_ParseTuple` function is declared as follows:: +For unpacking the tuple, CPython provides the :c:func:`PyArg_ParseTuple` +function, declared as follows:: int PyArg_ParseTuple(PyObject *arg, const char *format, ...); @@ -693,6 +386,19 @@ whose syntax is explained in :ref:`arg-parsing` in the Python/C API Reference Manual. The remaining arguments must be addresses of variables whose type is determined by the format string. +For example, to receive a single Python :py:class:`str` object and turn it +into a C buffer, you would use ``"s"`` as the format string:: + + const char *command; + if (!PyArg_ParseTuple(args, "s", &command)) { + return NULL; + } + +If an error is detected in the argument list, :c:func:`!PyArg_ParseTuple` +returns ``NULL`` (the error indicator for functions returning object pointers); +your function may return ``NULL``, relying on the exception set by +:c:func:`PyArg_ParseTuple`. + Note that while :c:func:`PyArg_ParseTuple` checks that the Python arguments have the required types, it cannot check the validity of the addresses of C variables passed to the call: if you make mistakes there, your code will probably crash or @@ -703,7 +409,6 @@ Note that any Python object references which are provided to the caller are Some example calls:: - #define PY_SSIZE_T_CLEAN #include :: @@ -773,6 +478,17 @@ Some example calls:: Keyword Parameters for Extension Functions ========================================== +If you also want your function to accept +:term:`keyword arguments `, use the :c:data:`METH_KEYWORDS` +flag in combination with :c:data:`METH_VARARGS`. +(:c:data:`!METH_KEYWORDS` can also be used with other flags; see its +documentation for the allowed combinations.) + +In this case, the C function should accept a third ``PyObject *`` parameter +which will be a dictionary of keywords. +Use :c:func:`PyArg_ParseTupleAndKeywords` to parse the arguments to such a +function. + .. index:: single: PyArg_ParseTupleAndKeywords (C function) The :c:func:`PyArg_ParseTupleAndKeywords` function is declared as follows:: @@ -833,19 +549,6 @@ Philbrick (philbrick@hks.com):: {NULL, NULL, 0, NULL} /* sentinel */ }; - static struct PyModuleDef keywdarg_module = { - .m_base = PyModuleDef_HEAD_INIT, - .m_name = "keywdarg", - .m_size = 0, - .m_methods = keywdarg_methods, - }; - - PyMODINIT_FUNC - PyInit_keywdarg(void) - { - return PyModuleDef_Init(&keywdarg_module); - } - .. _buildvalue: @@ -986,11 +689,11 @@ needed. Ownership of a reference can be transferred. There are three ways to dispose of an owned reference: pass it on, store it, or call :c:func:`Py_DECREF`. Forgetting to dispose of an owned reference creates a memory leak. -It is also possible to :dfn:`borrow` [#]_ a reference to an object. The +It is also possible to :dfn:`borrow` [#borrow]_ a reference to an object. The borrower of a reference should not call :c:func:`Py_DECREF`. The borrower must not hold on to the object longer than the owner from which it was borrowed. Using a borrowed reference after the owner has disposed of it risks using freed -memory and should be avoided completely [#]_. +memory and should be avoided completely [#dont-check-refcount]_. The advantage of borrowing over owning a reference is that you don't need to take care of disposing of the reference on all possible paths through the code @@ -1169,7 +872,7 @@ checking. The C function calling mechanism guarantees that the argument list passed to C functions (``args`` in the examples) is never ``NULL`` --- in fact it guarantees -that it is always a tuple [#]_. +that it is always a tuple [#old-calling-convention]_. It is a severe error to ever let a ``NULL`` pointer "escape" to the Python user. @@ -1200,9 +903,6 @@ define this symbol). Providing a C API for an Extension Module ========================================= -.. sectionauthor:: Konrad Hinsen - - Many extension modules just provide new functions and types to be used from Python, but sometimes the code in an extension module can be useful for other extension modules. For example, an extension module could implement a type @@ -1226,8 +926,8 @@ the module whose functions one wishes to call might not have been loaded yet! Portability therefore requires not to make any assumptions about symbol visibility. This means that all symbols in extension modules should be declared ``static``, except for the module's initialization function, in order to -avoid name clashes with other extension modules (as discussed in section -:ref:`methodtable`). And it means that symbols that *should* be accessible from +avoid name clashes with other extension modules. And it means that symbols +that *should* be accessible from other extension modules must be exported in a different way. Python provides a special mechanism to pass C-level information (pointers) from @@ -1269,8 +969,9 @@ file corresponding to the module provides a macro that takes care of importing the module and retrieving its C API pointers; client modules only have to call this macro before accessing the C API. -The exporting module is a modification of the :mod:`!spam` module from section -:ref:`extending-simpleexample`. The function :func:`!spam.system` does not call +The exporting module is a modification of the :mod:`!spam` module from the +:ref:`tutorial `. +The function :func:`!spam.system` does not call the C library function :c:func:`system` directly, but a function :c:func:`!PySpam_System`, which would of course do something more complicated in reality (such as adding "spam" to every command). This function @@ -1412,15 +1113,14 @@ code distribution). .. rubric:: Footnotes -.. [#] An interface for this function already exists in the standard module :mod:`os` - --- it was chosen as a simple and straightforward example. +.. [#borrow] The metaphor of "borrowing" a reference is not completely correct: + the owner still has a copy of the reference. -.. [#] The metaphor of "borrowing" a reference is not completely correct: the owner - still has a copy of the reference. - -.. [#] Checking that the reference count is at least 1 **does not work** --- the +.. [#dont-check-refcount] Checking that the reference count is at least 1 + **does not work** --- the reference count itself could be in freed memory and may thus be reused for another object! -.. [#] These guarantees don't hold when you use the "old" style calling convention --- +.. [#old-calling-convention] These guarantees don't hold when you use the + "old" style calling convention --- this is still found in much existing code. diff --git a/Doc/extending/first-extension-module.rst b/Doc/extending/first-extension-module.rst new file mode 100644 index 00000000000..cd755a98f7f --- /dev/null +++ b/Doc/extending/first-extension-module.rst @@ -0,0 +1,694 @@ +.. highlight:: c + + +.. _extending-simpleexample: +.. _first-extension-module: + +********************************* +Your first C API extension module +********************************* + +This tutorial will take you through creating a simple +Python extension module written in C or C++. + +We will use the low-level Python C API directly. +For easier ways to create extension modules, see +the :ref:`recommended third party tools `. + +The tutorial assumes basic knowledge about Python: you should be able to +define functions in Python code before starting to write them in C. +See :ref:`tutorial-index` for an introduction to Python itself. + +The tutorial should be approachable for anyone who can write a basic C library. +While we will mention several concepts that a C beginner would not be expected +to know, like ``static`` functions or linkage declarations, understanding these +is not necessary for success. + +We will focus on giving you a "feel" of what Python's C API is like. +It will not teach you important concepts, like error handling +and reference counting, which are covered in later chapters. + +We will assume that you use a Unix-like system (including macOS and +Linux), or Windows. +On other systems, you might need to adjust some details -- for example, +a system command name. + +You need to have a suitable C compiler and Python development headers installed. +On Linux, headers are often in a package like ``python3-dev`` +or ``python3-devel``. + +You need to be able to install Python packages. +This tutorial uses `pip `__ (``pip install``), but you +can substitute any tool that can build and install ``pyproject.toml``-based +projects, like `uv `_ (``uv pip install``). +Preferably, have a :ref:`virtual environment ` activated. + + +.. note:: + + This tutorial uses APIs that were added in CPython 3.15. + To create an extension that's compatible with earlier versions of CPython, + please follow an earlier version of this documentation. + + This tutorial uses C syntax added in C11 and C++20. + If your extension needs to be compatible with earlier standards, + please follow tutorials in documentation for Python 3.14 or below. + + +What we'll do +============= + +Let's create an extension module called ``spam`` [#why-spam]_, +which will include a Python interface to the C +standard library function :c:func:`system`. +This function is defined in ``stdlib.h``. +It takes a C string as argument, runs the argument as a system +command, and returns a result value as an integer. +A manual page for :c:func:`system` might summarize it this way:: + + #include + int system(const char *command); + +Note that like many functions in the C standard library, +this function is already exposed in Python. +In production, use :py:func:`os.system` or :py:func:`subprocess.run` +rather than the module you'll write here. + +We want this function to be callable from Python as follows: + +.. code-block:: pycon + + >>> import spam + >>> status = spam.system("whoami") + User Name + >>> status + 0 + +.. note:: + + The system command ``whoami`` prints out your username. + It's useful in tutorials like this one because it has the same name on + both Unix and Windows. + + +Start with the headers +====================== + +Begin by creating a directory for this tutorial, and switching to it +on the command line. +Then, create a file named :file:`spammodule.c` in your directory. +[#why-spammodule]_ + +In this file, we'll include two headers: :file:`Python.h` to pull in +all declarations of the Python C API, and :file:`stdlib.h` for the +:c:func:`system` function. [#stdlib-h]_ + +Add the following lines to :file:`spammodule.c`: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-at: + :end-at: + +Be sure to put :file:`stdlib.h`, and any other standard library includes, +*after* :file:`Python.h`. +On some systems, Python may define some pre-processor definitions +that affect the standard headers. + + +Running your build tool +======================= + +With only the includes in place, your extension won't do anything. +Still, it's a good time to compile it and try to import it. +This will ensure that your build tool works, so that you can make +and test incremental changes as you follow the rest of the text. + +CPython itself does not come with a tool to build extension modules; +it is recommended to use a third-party project for this. +In this tutorial, we'll use `meson-python`_. +(If you want to use another one, see :ref:`first-extension-other-tools`.) + +.. at the time of writing, meson-python has the least overhead for a + simple extension using PyModExport. + Change this if another tool makes things easier. + +``meson-python`` requires defining a "project" using two extra files. + +First, add ``pyproject.toml`` with these contents: + +.. code-block:: toml + + [build-system] + build-backend = 'mesonpy' + requires = ['meson-python'] + + [project] + # Placeholder project information + # (change this before distributing the module) + name = 'sampleproject' + version = '0' + +Then, create ``meson.build`` containing the following: + +.. code-block:: meson + + project('sampleproject', 'c') + + py = import('python').find_installation(pure: false) + + py.extension_module( + 'spam', # name of the importable Python module + 'spammodule.c', # the C source file + install: true, + ) + +.. note:: + + See `meson-python documentation `_ for details on + configuration. + +Now, build install the *project in the current directory* (``.``) via ``pip``: + +.. code-block:: sh + + python -m pip -v install . + +The ``-v`` (``--verbose``) option causes ``pip`` to show the output from +the compiler, which is often useful during development. + +.. tip:: + + If you don't have ``pip`` installed, run ``python -m ensurepip``, + preferably in a :ref:`virtual environment `. + (Or, if you prefer another tool that can build and install + ``pyproject.toml``-based projects, use that.) + +.. _meson-python: https://mesonbuild.com/meson-python/ +.. _virtual environment: https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments + +Note that you will need to run this command again every time you change your +extension. +Unlike Python, C has an explicit compilation step. + +When your extension is compiled and installed, start Python and try to +import it. +This should fail with the following exception: + +.. code-block:: pycon + + >>> import spam + Traceback (most recent call last): + ... + ImportError: dynamic module does not define module export function (PyModExport_spam or PyInit_spam) + + +Module export hook +================== + +The exception you got when you tried to import the module told you that Python +is looking for a "module export function", also known as a +:ref:`module export hook `. +Let's define one. + +First, add a prototype below the ``#include`` lines: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-after: /// Export hook prototype + :end-before: /// + +.. tip:: + The prototype is not strictly necessary, but some modern compilers emit + warnings without it. + It's generally better to add the prototype than to disable the warning. + +The :c:macro:`PyMODEXPORT_FUNC` macro declares the function's +return type, and adds any special linkage declarations needed +to make the function visible and usable when CPython loads it. + +After the prototype, add the function itself. +For now, make it return ``NULL``: + +.. code-block:: c + + PyMODEXPORT_FUNC + PyModExport_spam(void) + { + return NULL; + } + +Compile and load the module again. +You should get a different error this time. + +.. code-block:: pycon + + >>> import spam + Traceback (most recent call last): + ... + SystemError: module export hook for module 'spam' failed without setting an exception + +Simply returning ``NULL`` is *not* correct behavior for an export hook, +and CPython complains about it. +That's good -- it means that CPython found the function! +Let's now make it do something useful. + + +The slot table +============== + +Rather than ``NULL``, the export hook should return the information needed to +create a module. +Let's start with the basics: the name and docstring. + +The information should be defined in a ``static`` array of +:c:type:`PyModuleDef_Slot` entries, which are essentially key-value pairs. +Define this array just before your export hook: + +.. code-block:: c + + PyABIInfo_VAR(abi_info); + + static PyModuleDef_Slot spam_slots[] = { + {Py_mod_abi, &abi_info}, + {Py_mod_name, "spam"}, + {Py_mod_doc, "A wonderful module with an example function"}, + {0, NULL} + }; + +The ``PyABIInfo_VAR(abi_info);`` macro and the :c:data:`Py_mod_abi` slot +are a bit of boilerplate that helps prevent extensions compiled for +a different version of Python from crashing the interpreter. + +For both :c:data:`Py_mod_name` and :c:data:`Py_mod_doc`, the values are C +strings -- that is, NUL-terminated, UTF-8 encoded byte arrays. + +Note the zero-filled sentinel entry at the end. +If you forget it, you'll trigger undefined behavior. + +The array is defined as ``static`` -- that is, not visible outside this ``.c`` file. +This will be a common theme. +CPython only needs to access the export hook; all global variables +and all other functions should generally be ``static``, so that they don't +clash with other extensions. + +Return this array from your export hook instead of ``NULL``: + +.. code-block:: c + :emphasize-lines: 4 + + PyMODEXPORT_FUNC + PyModExport_spam(void) + { + return spam_slots; + } + +Now, recompile and try it out: + +.. code-block:: pycon + + >>> import spam + >>> print(spam) + + +You have an extension module! +Try ``help(spam)`` to see the docstring. + +The next step will be adding a function. + + +.. _backtoexample: + +Exposing a function +=================== + +To expose the :c:func:`system` C function directly to Python, +we'll need to write a layer of glue code to convert arguments from Python +objects to C values, and the C return value back to Python. + +One of the simplest ways to write glue code is a ":c:data:`METH_O`" function, +which takes two Python objects and returns one. +All Python objects -- regardless of the Python type -- are represented in C +as pointers to the :c:type:`PyObject` structure. + +Add such a function above the slots array:: + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + Py_RETURN_NONE; + } + +For now, we ignore the arguments, and use the :c:macro:`Py_RETURN_NONE` +macro, which expands to a ``return`` statement that properly returns +a Python :py:data:`None` object. + +Recompile your extension to make sure you don't have syntax errors. +We haven't yet added ``spam_system`` to the module, so you might get a +warning that ``spam_system`` is unused. + +.. _methodtable: + +Method definitions +------------------ + +To expose the C function to Python, you will need to provide several pieces of +information in a structure called +:c:type:`PyMethodDef` [#why-pymethoddef]_: + +* ``ml_name``: the name of the Python function; +* ``ml_doc``: a docstring; +* ``ml_meth``: the C function to be called; and +* ``ml_flags``: a set of flags describing details like how Python arguments are + passed to the C function. + We'll use :c:data:`METH_O` here -- the flag that matches our + ``spam_system`` function's signature. + +Because modules typically create several functions, these definitions +need to be collected in an array, with a zero-filled sentinel at the end. +Add this array just below the ``spam_system`` function: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-after: /// Module method table + :end-before: /// + +As with module slots, a zero-filled sentinel marks the end of the array. + +Next, we'll add the method to the module. +Add a :c:data:`Py_mod_methods` slot to your :c:type:`PyMethodDef` array: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-after: /// Module slot table + :end-before: /// + :emphasize-lines: 5 + +Recompile your extension again, and test it. +Be sure to restart the Python interpreter, so that ``import spam`` picks +up the new version of the module. + +You should now be able to call the function: + +.. code-block:: pycon + + >>> import spam + >>> print(spam.system) + + >>> print(spam.system('whoami')) + None + +Note that our ``spam.system`` does not yet run the ``whoami`` command; +it only returns ``None``. + +Check that the function accepts exactly one argument, as specified by +the :c:data:`METH_O` flag: + +.. code-block:: pycon + + >>> print(spam.system('too', 'many', 'arguments')) + Traceback (most recent call last): + ... + TypeError: spam.system() takes exactly one argument (3 given) + + +Returning an integer +==================== + +Now, let's take a look at the return value. +Instead of ``None``, we'll want ``spam.system`` to return a number -- that is, +a Python :py:type:`int` object. +Eventually this will be the exit code of a system command, +but let's start with a fixed value, say, ``3``. + +The Python C API provides a function to create a Python :py:type:`int` object +from a C ``int`` value: :c:func:`PyLong_FromLong`. [#why-pylongfromlong]_ + +To call it, replace the ``Py_RETURN_NONE`` with the following 3 lines: + +.. this could be a one-liner, but we want to show the data types here + +.. code-block:: c + :emphasize-lines: 4-6 + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + int status = 3; + PyObject *result = PyLong_FromLong(status); + return result; + } + + +Recompile, restart the Python interpreter again, +and check that the function now returns 3: + +.. code-block:: pycon + + >>> import spam + >>> spam.system('whoami') + 3 + + +Accepting a string +================== + +Finally, let's handle the function argument. + +Our C function, :c:func:`!spam_system`, takes two arguments. +The first one, ``PyObject *self``, will be set to the ``spam`` module +object. +This isn't useful in our case, so we'll ignore it. + +The other one, ``PyObject *arg``, will be set to the object that the user +passed from Python. +We expect that it should be a Python string. +In order to use the information in it, we will need +to convert it to a C value -- in this case, a C string (``const char *``). + +There's a slight type mismatch here: Python's :py:class:`str` objects store +Unicode text, but C strings are arrays of bytes. +So, we'll need to *encode* the data, and we'll use the UTF-8 encoding for it. +(UTF-8 might not always be correct for system commands, but it's what +:py:meth:`str.encode` uses by default, +and the C API has special support for it.) + +The function to encode a Python string into a UTF-8 buffer is named +:c:func:`PyUnicode_AsUTF8AndSize` [#why-pyunicodeasutf8]_. +Call it like this: + +.. code-block:: c + :emphasize-lines: 4 + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + const char *command = PyUnicode_AsUTF8AndSize(arg, NULL); + int status = 3; + PyObject *result = PyLong_FromLong(status); + return result; + } + +If :c:func:`PyUnicode_AsUTF8AndSize` is successful, *command* will point to the +resulting C string -- a zero-terminated array of bytes [#embedded-nul]_. +This buffer is managed by the *arg* object, which means we don't need to free +it, but we must follow some rules: + +* We should only use the buffer inside the ``spam_system`` function. + After ``spam_system`` returns, *arg* and the buffer it manages might be + garbage-collected. +* We must not modify it. This is why we use ``const``. + +If :c:func:`PyUnicode_AsUTF8AndSize` was *not* successful, it returns a ``NULL`` +pointer. +When calling *any* Python C API, we always need to handle such error cases. +The way to do this in general is left for later chapters of this documentation. +For now, be assured that we are already handling errors from +:c:func:`PyLong_FromLong` correctly. + +For the :c:func:`PyUnicode_AsUTF8AndSize` call, the correct way to handle +errors is returning ``NULL`` from ``spam_system``. +Add an ``if`` block for this: + + +.. code-block:: c + :emphasize-lines: 5-7 + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + const char *command = PyUnicode_AsUTF8AndSize(arg); + if (command == NULL) { + return NULL; + } + int status = 3; + PyObject *result = PyLong_FromLong(status); + return result; + } + +To test that error handling works, compile again, restart Python so that +``import spam`` picks up the new version of your module, and try passing +a non-string value to your function: + +.. code-block:: pycon + + >>> import spam + >>> spam.system(3) + Traceback (most recent call last): + ... + TypeError: bad argument type for built-in operation + +Now, all that is left is calling the C library function :c:func:`system` with +the ``char *`` buffer, and using its result instead of the ``3``: + +.. code-block:: c + :emphasize-lines: 8 + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + const char *command = PyUnicode_AsUTF8AndSize(arg); + if (command == NULL) { + return NULL; + } + int status = system(command); + PyObject *result = PyLong_FromLong(status); + return result; + } + +Compile your module, restart Python, and test. +This time, you should see your username -- the output of the ``whoami`` +system command: + +.. code-block:: pycon + + >>> import spam + >>> result = spam.system('whoami') + User Name + >>> result + 0 + +You can also test with other commands, like ``ls``, ``dir``, or one +that doesn't exist: + +.. code-block:: pycon + + >>> import spam + >>> result = spam.system('nonexistent-command') + sh: line 1: nonexistent-command: command not found + >>> result + 32512 + + +The result +========== + + +Congratulations! +You have written a complete Python C API extension module, +and completed this tutorial! + +Here is the entire source file, for your convenience: + +.. _extending-spammodule-source: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-at: /// + + +.. _first-extension-other-tools: + +Appendix: Other build tools +=========================== + +You should be able to follow this tutorial -- except the +*Running your build tool* section itself -- with a build tool other +than ``meson-python``. + +The Python Packaging User Guide has a `list of recommended tools `_; +be sure to choose one for the C language. + + +Workaround for missing PyInit function +-------------------------------------- + +If your build tool output complains about missing ``PyInit_spam``, +add the following function to your module for now: + +.. code-block:: c + + // A workaround + void *PyInit_spam(void) { return NULL; } + +This is a shim for an old-style :ref:`initialization function `, +which was required in extension modules for CPython 3.14 and below. +Current CPython does not need it, but some build tools may still assume that +all extension modules need to define it. + +If you use this workaround, you will get the exception +``SystemError: initialization of spam failed without raising an exception`` +instead of +``ImportError: dynamic module does not define module export function``. + + +Compiling directly +------------------ + +Using a third-party build tool is heavily recommended, +as it will take care of various details of your platform and Python +installation, of naming the resulting extension, and, later, of distributing +your work. + +If you are building an extension for as *specific* system, or for yourself +only, you might instead want to run your compiler directly. +The way to do this is system-specific; be prepared for issues you will need +to solve yourself. + +Linux +^^^^^ + +On Linux, the Python development package may include a ``python3-config`` +command that prints out the required compiler flags. +If you use it, check that it corresponds to the CPython interpreter you'll use +to load the module. +Then, start with the following command: + +.. code-block:: sh + + gcc --shared $(python3-config --cflags --ldflags) spammodule.c -o spam.so + +This should generate a ``spam.so`` file that you need to put in a directory +on :py:attr:`sys.path`. + + +.. rubric:: Footnotes + +.. [#why-spam] ``spam`` is the favorite food of Monty Python fans... +.. [#why-spammodule] The source file name is entirely up to you, + though some tools can be picky about the ``.c`` extension. + This tutorial uses the traditional ``*module.c`` suffix. + Some people would just use :file:`spam.c` to implement a module + named ``spam``, + projects where Python isn't the primary language might use ``py_spam.c``, + and so on. +.. [#stdlib-h] Including :file:`stdlib.h` is technically not necessary, + since :file:`Python.h` includes it and + :ref:`several other standard headers ` for its own use + or for backwards compatibility. + However, it is good practice to explicitly include what you need. +.. [#why-pymethoddef] The :c:type:`!PyMethodDef` structure is also used + to create methods of classes, so there's no separate + ":c:type:`!PyFunctionDef`". +.. [#why-pylongfromlong] The name :c:func:`PyLong_FromLong` + might not seem obvious. + ``PyLong`` refers to a the Python :py:class:`int`, which was originally + called ``long``; the ``FromLong`` refers to the C ``long`` (or ``long int``) + type. +.. [#why-pyunicodeasutf8] Here, ``PyUnicode`` refers to the original name of + the Python :py:class:`str` class: ``unicode``. + + The ``AndSize`` part of the name refers to the fact that this function can + also retrieve the size of the buffer, using an output argument. + We don't need this, so we set the second argument to NULL. +.. [#embedded-nul] We're ignoring the fact that Python strings can also + contain NUL bytes, which terminate a C string. + In other words, our function will treat ``spam.system("foo\0bar")`` as + ``spam.system("foo")``. + This possibility can lead to security issues, so the real ``os.system`` + function size checks for this case and raises an error. diff --git a/Doc/extending/index.rst b/Doc/extending/index.rst index 4cc2c96d8d5..c0c494c3059 100644 --- a/Doc/extending/index.rst +++ b/Doc/extending/index.rst @@ -5,15 +5,17 @@ ################################################## This document describes how to write modules in C or C++ to extend the Python -interpreter with new modules. Those modules can not only define new functions -but also new object types and their methods. The document also describes how +interpreter with new modules. Those modules can do what Python code does -- +define functions, object types and methods -- but also interact with native +libraries or achieve better performance by avoiding the overhead of an +interpreter. The document also describes how to embed the Python interpreter in another application, for use as an extension language. Finally, it shows how to compile and link extension modules so that they can be loaded dynamically (at run time) into the interpreter, if the underlying operating system supports this feature. -This document assumes basic knowledge about Python. For an informal -introduction to the language, see :ref:`tutorial-index`. :ref:`reference-index` +This document assumes basic knowledge about C and Python. For an informal +introduction to Python, see :ref:`tutorial-index`. :ref:`reference-index` gives a more formal definition of the language. :ref:`library-index` documents the existing object types, functions and modules (both built-in and written in Python) that give the language its wide application range. @@ -21,37 +23,75 @@ Python) that give the language its wide application range. For a detailed description of the whole Python/C API, see the separate :ref:`c-api-index`. +To support extensions, Python's C API (Application Programmers Interface) +defines a set of functions, macros and variables that provide access to most +aspects of the Python run-time system. The Python API is incorporated in a C +source file by including the header ``"Python.h"``. + +.. note:: + + The C extension interface is specific to CPython, and extension modules do + not work on other Python implementations. In many cases, it is possible to + avoid writing C extensions and preserve portability to other implementations. + For example, if your use case is calling C library functions or system calls, + you should consider using the :mod:`ctypes` module or the `cffi + `_ library rather than writing + custom C code. + These modules let you write Python code to interface with C code and are more + portable between implementations of Python than writing and compiling a C + extension module. + + +.. toctree:: + :hidden: + + first-extension-module.rst + extending.rst + newtypes_tutorial.rst + newtypes.rst + building.rst + windows.rst + embedding.rst + Recommended third party tools ============================= -This guide only covers the basic tools for creating extensions provided +This document only covers the basic tools for creating extensions provided as part of this version of CPython. Some :ref:`third party tools ` offer both simpler and more sophisticated approaches to creating C and C++ extensions for Python. +While this document is aimed at extension authors, it should also be helpful to +the authors of such tools. +For example, the tutorial module can serve as a simple test case for a build +tool or sample expected output of a code generator. -Creating extensions without third party tools -============================================= + +C API Tutorial +============== + +This tutorial describes how to write a simple module in C or C++, +using the Python C API -- that is, using the basic tools provided +as part of this version of CPython. + + +#. :ref:`first-extension-module` + + +Guides for intermediate topics +============================== This section of the guide covers creating C and C++ extensions without assistance from third party tools. It is intended primarily for creators of those tools, rather than being a recommended way to create your own C extensions. -.. seealso:: - - :pep:`489` -- Multi-phase extension module initialization - -.. toctree:: - :maxdepth: 2 - :numbered: - - extending.rst - newtypes_tutorial.rst - newtypes.rst - building.rst - windows.rst +* :ref:`extending-intro` +* :ref:`defining-new-types` +* :ref:`new-types-topics` +* :ref:`building` +* :ref:`building-on-windows` Embedding the CPython runtime in a larger application ===================================================== @@ -61,8 +101,4 @@ interpreter as the main application, it is desirable to instead embed the CPython runtime inside a larger application. This section covers some of the details involved in doing that successfully. -.. toctree:: - :maxdepth: 2 - :numbered: - - embedding.rst +* :ref:`embedding` diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index e3612f3a187..26085b5cebd 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -560,6 +560,8 @@ For an object to be weakly referenceable, the extension type must set the field. The legacy :c:member:`~PyTypeObject.tp_weaklistoffset` field should be left as zero. +If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` should also be set. + Concretely, here is how the statically declared type object would look:: static PyTypeObject TrivialType = { diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index 3bbee33bd50..9f3cd1d6f4c 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -6,11 +6,6 @@ Defining Extension Types: Tutorial ********************************** -.. sectionauthor:: Michael Hudson -.. sectionauthor:: Dave Kuhlman -.. sectionauthor:: Jim Fulton - - Python allows the writer of a C extension module to define new types that can be manipulated from Python code, much like the built-in :class:`str` and :class:`list` types. The code for all extension types follows a diff --git a/Doc/extending/windows.rst b/Doc/extending/windows.rst index a97c6182553..cd81b443603 100644 --- a/Doc/extending/windows.rst +++ b/Doc/extending/windows.rst @@ -47,9 +47,6 @@ things manually, it may be instructive to study the project file for the Differences Between Unix and Windows ==================================== -.. sectionauthor:: Chris Phoenix - - Unix and Windows use completely different paradigms for run-time loading of code. Before you try to build a module that can be dynamically loaded, be aware of how your system works. @@ -109,9 +106,6 @@ separate copy. Using DLLs in Practice ====================== -.. sectionauthor:: Chris Phoenix - - Windows Python is built in Microsoft Visual C++; using other compilers may or may not work. The rest of this section is MSVC++ specific. diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 578777d7f23..698f2117ff5 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -186,7 +186,7 @@ How do I get documentation on Python? ------------------------------------- The standard documentation for the current stable version of Python is available -at https://docs.python.org/3/. PDF, plain text, and downloadable HTML versions are +at https://docs.python.org/3/. EPUB, plain text, and downloadable HTML versions are also available at https://docs.python.org/3/download.html. The documentation is written in reStructuredText and processed by `the Sphinx diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 6f9dfa8616e..591565cbc01 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -8,11 +8,11 @@ Programming FAQ .. contents:: -General Questions +General questions ================= -Is there a source code level debugger with breakpoints, single-stepping, etc.? ------------------------------------------------------------------------------- +Is there a source code-level debugger with breakpoints and single-stepping? +--------------------------------------------------------------------------- Yes. @@ -25,8 +25,7 @@ Reference Manual `. You can also write your own debugger by using the code for pdb as an example. The IDLE interactive development environment, which is part of the standard -Python distribution (normally available as -`Tools/scripts/idle3 `_), +Python distribution (normally available as :mod:`idlelib`), includes a graphical debugger. PythonWin is a Python IDE that includes a GUI debugger based on pdb. The @@ -48,7 +47,6 @@ There are a number of commercial Python IDEs that include graphical debuggers. They include: * `Wing IDE `_ -* `Komodo IDE `_ * `PyCharm `_ @@ -57,13 +55,15 @@ Are there tools to help find bugs or perform static analysis? Yes. -`Pylint `_ and -`Pyflakes `_ do basic checking that will +`Ruff `__, +`Pylint `__ and +`Pyflakes `__ do basic checking that will help you catch bugs sooner. -Static type checkers such as `Mypy `_, -`Pyre `_, and -`Pytype `_ can check type hints in Python +Static type checkers such as `mypy `__, +`ty `__, +`Pyrefly `__, and +`pytype `__ can check type hints in Python source code. @@ -79,7 +79,7 @@ set of modules required by a program and bind these modules together with a Python binary to produce a single executable. One is to use the freeze tool, which is included in the Python source tree as -`Tools/freeze `_. +:source:`Tools/freeze`. It converts Python byte code to C arrays; with a C compiler you can embed all your modules into a new program, which is then linked with the standard Python modules. @@ -103,6 +103,7 @@ executables: * `py2app `_ (macOS only) * `py2exe `_ (Windows only) + Are there coding standards or a style guide for Python programs? ---------------------------------------------------------------- @@ -110,7 +111,7 @@ Yes. The coding style required for standard library modules is documented as :pep:`8`. -Core Language +Core language ============= .. _faq-unboundlocalerror: @@ -143,7 +144,7 @@ results in an :exc:`!UnboundLocalError`: >>> foo() Traceback (most recent call last): ... - UnboundLocalError: local variable 'x' referenced before assignment + UnboundLocalError: cannot access local variable 'x' where it is not associated with a value This is because when you make an assignment to a variable in a scope, that variable becomes local to that scope and shadows any similarly named variable @@ -208,7 +209,7 @@ Why do lambdas defined in a loop with different values all return the same resul ---------------------------------------------------------------------------------- Assume you use a for loop to define a few different lambdas (or even plain -functions), e.g.:: +functions), for example:: >>> squares = [] >>> for x in range(5): @@ -227,7 +228,7 @@ they all return ``16``:: This happens because ``x`` is not local to the lambdas, but is defined in the outer scope, and it is accessed when the lambda is called --- not when it is defined. At the end of the loop, the value of ``x`` is ``4``, so all the -functions now return ``4**2``, i.e. ``16``. You can also verify this by +functions now return ``4**2``, that is ``16``. You can also verify this by changing the value of ``x`` and see how the results of the lambdas change:: >>> x = 8 @@ -298,9 +299,9 @@ using multiple imports per line uses less screen space. It's good practice if you import modules in the following order: -1. standard library modules -- e.g. :mod:`sys`, :mod:`os`, :mod:`argparse`, :mod:`re` +1. standard library modules -- such as :mod:`sys`, :mod:`os`, :mod:`argparse`, :mod:`re` 2. third-party library modules (anything installed in Python's site-packages - directory) -- e.g. :mod:`!dateutil`, :mod:`!requests`, :mod:`!PIL.Image` + directory) -- such as :pypi:`dateutil`, :pypi:`requests`, :pypi:`tzdata` 3. locally developed modules It is sometimes necessary to move imports to a function or class to avoid @@ -494,11 +495,11 @@ new objects). In other words: -* If we have a mutable object (:class:`list`, :class:`dict`, :class:`set`, - etc.), we can use some specific operations to mutate it and all the variables +* If we have a mutable object (such as :class:`list`, :class:`dict`, :class:`set`), + we can use some specific operations to mutate it and all the variables that refer to it will see the change. -* If we have an immutable object (:class:`str`, :class:`int`, :class:`tuple`, - etc.), all the variables that refer to it will always see the same value, +* If we have an immutable object (such as :class:`str`, :class:`int`, :class:`tuple`), + all the variables that refer to it will always see the same value, but operations that transform that value into a new value always return a new object. @@ -511,7 +512,7 @@ How do I write a function with output parameters (call by reference)? Remember that arguments are passed by assignment in Python. Since assignment just creates references to objects, there's no alias between an argument name in -the caller and callee, and so no call-by-reference per se. You can achieve the +the caller and callee, and consequently no call-by-reference. You can achieve the desired effect in a number of ways. 1) By returning a tuple of the results:: @@ -714,8 +715,8 @@ not:: "a" in ("b", "a") -The same is true of the various assignment operators (``=``, ``+=`` etc). They -are not truly operators but syntactic delimiters in assignment statements. +The same is true of the various assignment operators (``=``, ``+=``, and so on). +They are not truly operators but syntactic delimiters in assignment statements. Is there an equivalent of C's "?:" ternary operator? @@ -868,9 +869,9 @@ with either a space or parentheses. How do I convert a string to a number? -------------------------------------- -For integers, use the built-in :func:`int` type constructor, e.g. ``int('144') +For integers, use the built-in :func:`int` type constructor, for example, ``int('144') == 144``. Similarly, :func:`float` converts to a floating-point number, -e.g. ``float('144') == 144.0``. +for example, ``float('144') == 144.0``. By default, these interpret the number as decimal, so that ``int('0144') == 144`` holds true, and ``int('0x144')`` raises :exc:`ValueError`. ``int(string, @@ -887,18 +888,18 @@ unwanted side effects. For example, someone could pass directory. :func:`eval` also has the effect of interpreting numbers as Python expressions, -so that e.g. ``eval('09')`` gives a syntax error because Python does not allow +so that, for example, ``eval('09')`` gives a syntax error because Python does not allow leading '0' in a decimal number (except '0'). How do I convert a number to a string? -------------------------------------- -To convert, e.g., the number ``144`` to the string ``'144'``, use the built-in type +For example, to convert the number ``144`` to the string ``'144'``, use the built-in type constructor :func:`str`. If you want a hexadecimal or octal representation, use the built-in functions :func:`hex` or :func:`oct`. For fancy formatting, see -the :ref:`f-strings` and :ref:`formatstrings` sections, -e.g. ``"{:04d}".format(144)`` yields +the :ref:`f-strings` and :ref:`formatstrings` sections. +For example, ``"{:04d}".format(144)`` yields ``'0144'`` and ``"{:.3f}".format(1.0/3.0)`` yields ``'0.333'``. @@ -908,7 +909,7 @@ How do I modify a string in place? You can't, because strings are immutable. In most situations, you should simply construct a new string from the various parts you want to assemble it from. However, if you need an object with the ability to modify in-place -unicode data, try using an :class:`io.StringIO` object or the :mod:`array` +Unicode data, try using an :class:`io.StringIO` object or the :mod:`array` module:: >>> import io @@ -1066,13 +1067,14 @@ the raw string:: Also see the specification in the :ref:`language reference `. + Performance =========== My program is too slow. How do I speed it up? --------------------------------------------- -That's a tough one, in general. First, here are a list of things to +That's a tough one, in general. First, here is a list of things to remember before diving further: * Performance characteristics vary across Python implementations. This FAQ @@ -1125,6 +1127,7 @@ yourself. The wiki page devoted to `performance tips `_. + .. _efficient_string_concatenation: What is the most efficient way to concatenate many strings together? @@ -1143,7 +1146,7 @@ them into a list and call :meth:`str.join` at the end:: chunks.append(s) result = ''.join(chunks) -(another reasonably efficient idiom is to use :class:`io.StringIO`) +(Another reasonably efficient idiom is to use :class:`io.StringIO`.) To accumulate many :class:`bytes` objects, the recommended idiom is to extend a :class:`bytearray` object using in-place concatenation (the ``+=`` operator):: @@ -1153,7 +1156,7 @@ a :class:`bytearray` object using in-place concatenation (the ``+=`` operator):: result += b -Sequences (Tuples/Lists) +Sequences (tuples/lists) ======================== How do I convert between tuples and lists? @@ -1217,8 +1220,8 @@ list, deleting duplicates as you go:: else: last = mylist[i] -If all elements of the list may be used as set keys (i.e. they are all -:term:`hashable`) this is often faster :: +If all elements of the list may be used as set keys (that is, they are all +:term:`hashable`) this is often faster:: mylist = list(set(mylist)) @@ -1226,13 +1229,13 @@ This converts the list into a set, thereby removing duplicates, and then back into a list. -How do you remove multiple items from a list --------------------------------------------- +How do you remove multiple items from a list? +--------------------------------------------- As with removing duplicates, explicitly iterating in reverse with a delete condition is one possibility. However, it is easier and faster to use slice replacement with an implicit or explicit forward iteration. -Here are three variations.:: +Here are three variations:: mylist[:] = filter(keep_function, mylist) mylist[:] = (x for x in mylist if keep_condition) @@ -1254,7 +1257,7 @@ difference is that a Python list can contain objects of many different types. The ``array`` module also provides methods for creating arrays of fixed types with compact representations, but they are slower to index than lists. Also note that `NumPy `_ -and other third party packages define array-like structures with +and other third-party packages define array-like structures with various characteristics as well. To get Lisp-style linked lists, you can emulate *cons cells* using tuples:: @@ -1324,7 +1327,7 @@ Or, you can use an extension that provides a matrix datatype; `NumPy How do I apply a method or function to a sequence of objects? ------------------------------------------------------------- -To call a method or function and accumulate the return values is a list, +To call a method or function and accumulate the return values in a list, a :term:`list comprehension` is an elegant solution:: result = [obj.method() for obj in mylist] @@ -1340,6 +1343,7 @@ a plain :keyword:`for` loop will suffice:: for obj in mylist: function(obj) + .. _faq-augmented-assignment-tuple-error: Why does a_tuple[i] += ['item'] raise an exception when the addition works? @@ -1444,7 +1448,7 @@ How can I sort one list by values from another list? ---------------------------------------------------- Merge them into an iterator of tuples, sort the resulting list, and then pick -out the element you want. :: +out the element you want. >>> list1 = ["what", "I'm", "sorting", "by"] >>> list2 = ["something", "else", "to", "sort"] @@ -1504,14 +1508,15 @@ How do I check if an object is an instance of a given class or of a subclass of Use the built-in function :func:`isinstance(obj, cls) `. You can check if an object is an instance of any of a number of classes by providing a tuple instead of a -single class, e.g. ``isinstance(obj, (class1, class2, ...))``, and can also -check whether an object is one of Python's built-in types, e.g. +single class, for example, ``isinstance(obj, (class1, class2, ...))``, and can also +check whether an object is one of Python's built-in types, for example, ``isinstance(obj, str)`` or ``isinstance(obj, (int, float, complex))``. Note that :func:`isinstance` also checks for virtual inheritance from an :term:`abstract base class`. So, the test will return ``True`` for a registered class even if hasn't directly or indirectly inherited from it. To -test for "true inheritance", scan the :term:`MRO` of the class: +test for "true inheritance", scan the :term:`method resolution order` (MRO) of +the class: .. testcode:: @@ -1574,7 +1579,7 @@ call it:: What is delegation? ------------------- -Delegation is an object oriented technique (also called a design pattern). +Delegation is an object-oriented technique (also called a design pattern). Let's say you have an object ``x`` and want to change the behaviour of just one of its methods. You can create a new class that provides a new implementation of the method you're interested in changing and delegates all other methods to @@ -1645,7 +1650,7 @@ How can I organize my code to make it easier to change the base class? You could assign the base class to an alias and derive from the alias. Then all you have to change is the value assigned to the alias. Incidentally, this trick -is also handy if you want to decide dynamically (e.g. depending on availability +is also handy if you want to decide dynamically (such as depending on availability of resources) which base class to use. Example:: class Base: @@ -1710,9 +1715,9 @@ How can I overload constructors (or methods) in Python? This answer actually applies to all methods, but the question usually comes up first in the context of constructors. -In C++ you'd write +In C++ you'd write: -.. code-block:: c +.. code-block:: c++ class C { C() { cout << "No arguments\n"; } @@ -1731,7 +1736,7 @@ default arguments. For example:: This is not entirely equivalent, but close enough in practice. -You could also try a variable-length argument list, e.g. :: +You could also try a variable-length argument list, for example:: def __init__(self, *args): ... @@ -1774,6 +1779,7 @@ to use private variable names at all. The :ref:`private name mangling specifications ` for details and special cases. + My class defines __del__ but it is not called when I delete the object. ----------------------------------------------------------------------- @@ -1783,7 +1789,7 @@ The :keyword:`del` statement does not necessarily call :meth:`~object.__del__` - decrements the object's reference count, and if this reaches zero :meth:`!__del__` is called. -If your data structures contain circular links (e.g. a tree where each child has +If your data structures contain circular links (for example, a tree where each child has a parent reference and each parent has a list of children) the reference counts will never go back to zero. Once in a while Python runs an algorithm to detect such cycles, but the garbage collector might run some time after the last @@ -1852,6 +1858,8 @@ to the object: 13891296 +.. _faq-identity-with-is: + When can I rely on identity tests with the *is* operator? --------------------------------------------------------- @@ -1883,9 +1891,9 @@ are preferred. In particular, identity tests should not be used to check constants such as :class:`int` and :class:`str` which aren't guaranteed to be singletons:: - >>> a = 1000 - >>> b = 500 - >>> c = b + 500 + >>> a = 10_000_000 + >>> b = 5_000_000 + >>> c = b + 5_000_000 >>> a is c False @@ -1916,7 +1924,7 @@ correctly using identity tests: .. code-block:: python - _sentinel = object() + _sentinel = sentinel('_sentinel') def pop(self, key, default=_sentinel): if key in self: @@ -1954,9 +1962,9 @@ parent class: .. testcode:: - from datetime import date + import datetime as dt - class FirstOfMonthDate(date): + class FirstOfMonthDate(dt.date): "Always choose the first day of the month" def __new__(cls, year, month, day): return super().__new__(cls, year, month, 1) @@ -1999,7 +2007,7 @@ The two principal tools for caching methods are former stores results at the instance level and the latter at the class level. -The *cached_property* approach only works with methods that do not take +The ``cached_property`` approach only works with methods that do not take any arguments. It does not create a reference to the instance. The cached method result will be kept only as long as the instance is alive. @@ -2008,7 +2016,7 @@ method result will be released right away. The disadvantage is that if instances accumulate, so too will the accumulated method results. They can grow without bound. -The *lru_cache* approach works with methods that have :term:`hashable` +The ``lru_cache`` approach works with methods that have :term:`hashable` arguments. It creates a reference to the instance unless special efforts are made to pass in weak references. @@ -2042,11 +2050,11 @@ This example shows the various techniques:: # Depends on the station_id, date, and units. The above example assumes that the *station_id* never changes. If the -relevant instance attributes are mutable, the *cached_property* approach +relevant instance attributes are mutable, the ``cached_property`` approach can't be made to work because it cannot detect changes to the attributes. -To make the *lru_cache* approach work when the *station_id* is mutable, +To make the ``lru_cache`` approach work when the *station_id* is mutable, the class needs to define the :meth:`~object.__eq__` and :meth:`~object.__hash__` methods so that the cache can detect relevant attribute updates:: @@ -2092,10 +2100,10 @@ one user but run as another, such as if you are testing with a web server. Unless the :envvar:`PYTHONDONTWRITEBYTECODE` environment variable is set, creation of a .pyc file is automatic if you're importing a module and Python -has the ability (permissions, free space, etc...) to create a ``__pycache__`` +has the ability (permissions, free space, and so on) to create a ``__pycache__`` subdirectory and write the compiled module to that subdirectory. -Running Python on a top level script is not considered an import and no +Running Python on a top-level script is not considered an import and no ``.pyc`` will be created. For example, if you have a top-level module ``foo.py`` that imports another module ``xyz.py``, when you run ``foo`` (by typing ``python foo.py`` as a shell command), a ``.pyc`` will be created for @@ -2114,7 +2122,7 @@ the ``compile()`` function in that module interactively:: This will write the ``.pyc`` to a ``__pycache__`` subdirectory in the same location as ``foo.py`` (or you can override that with the optional parameter -``cfile``). +*cfile*). You can also automatically compile all files in a directory or directories using the :mod:`compileall` module. You can do it from the shell prompt by running @@ -2219,7 +2227,7 @@ changed module, do this:: importlib.reload(modname) Warning: this technique is not 100% fool-proof. In particular, modules -containing statements like :: +containing statements like:: from modname import some_objects diff --git a/Doc/faq/python-video-icon.png b/Doc/faq/python-video-icon.png deleted file mode 100644 index 265da50c7b3..00000000000 Binary files a/Doc/faq/python-video-icon.png and /dev/null differ diff --git a/Doc/glossary.rst b/Doc/glossary.rst index c0ca0be304e..56bc799d945 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -39,10 +39,11 @@ Glossary ABCs with the :mod:`abc` module. annotate function - A function that can be called to retrieve the :term:`annotations ` - of an object. This function is accessible as the :attr:`~object.__annotate__` - attribute of functions, classes, and modules. Annotate functions are a - subset of :term:`evaluate functions `. + A callable that can be called to retrieve the :term:`annotations ` of + an object. Annotate functions are usually :term:`functions `, + automatically generated as the :attr:`~object.__annotate__` attribute of functions, + classes, and modules. Annotate functions are a subset of + :term:`evaluate functions `. annotation A label associated with a variable, a class @@ -134,6 +135,14 @@ Glossary iterator's :meth:`~object.__anext__` method until it raises a :exc:`StopAsyncIteration` exception. Introduced by :pep:`492`. + atomic operation + An operation that appears to execute as a single, indivisible step: no + other thread can observe it half-done, and its effects become visible all + at once. Python does not guarantee that high-level statements are atomic + (for example, ``x += 1`` performs multiple bytecode operations and is not + atomic). Atomicity is only guaranteed where explicitly documented. See + also :term:`race condition` and :term:`data race`. + attached thread state A :term:`thread state` that is active for the current OS thread. @@ -152,9 +161,9 @@ Glossary On most builds of Python, having an attached thread state implies that the caller holds the :term:`GIL` for the current interpreter, so only one OS thread can have an attached thread state at a given moment. In - :term:`free-threaded ` builds of Python, threads can concurrently - hold an attached thread state, allowing for true parallelism of the bytecode - interpreter. + :term:`free-threaded builds ` of Python, threads can + concurrently hold an attached thread state, allowing for true parallelism of + the bytecode interpreter. attribute A value associated with an object which is usually referenced by name @@ -289,6 +298,22 @@ Glossary advanced mathematical feature. If you're not aware of a need for them, it's almost certain you can safely ignore them. + concurrency + The ability of a computer program to perform multiple tasks at the same + time. Python provides libraries for writing programs that make use of + different forms of concurrency. :mod:`asyncio` is a library for dealing + with asynchronous tasks and coroutines. :mod:`threading` provides + access to operating system threads and :mod:`multiprocessing` to + operating system processes. Multi-core processors can execute threads and + processes on different CPU cores at the same time (see + :term:`parallelism`). + + concurrent modification + When multiple threads modify shared data at the same time. Concurrent + modification without proper synchronization can cause + :term:`race conditions `, and might also trigger a + :term:`data race `, data corruption, or both. + context This term has different meanings depending on where and how it is used. Some common meanings: @@ -363,6 +388,28 @@ Glossary the :term:`cyclic garbage collector ` is to identify these groups and break the reference cycles so that the memory can be reclaimed. + data race + A situation where multiple threads access the same memory location + concurrently, at least one of the accesses is a write, and the threads + do not use any synchronization to control their access. Data races + lead to :term:`non-deterministic` behavior and can cause data corruption. + Proper use of :term:`locks ` and other :term:`synchronization primitives + ` prevents data races. Note that data races + can only happen in native code, but that :term:`native code` might be + exposed in a Python API. See also :term:`race condition` and + :term:`thread-safe`. + + deadlock + A situation in which two or more tasks (threads, processes, or coroutines) + wait indefinitely for each other to release resources or complete actions, + preventing any from making progress. For example, if thread A holds lock + 1 and waits for lock 2, while thread B holds lock 2 and waits for lock 1, + both threads will wait indefinitely. In Python this often arises from + acquiring multiple locks in conflicting orders or from circular + join/await dependencies. Deadlocks can be avoided by always acquiring + multiple :term:`locks ` in a consistent order. See also + :term:`lock` and :term:`reentrant`. + decorator A function returning another function, usually applied as a function transformation using the ``@wrapper`` syntax. Common examples for @@ -534,6 +581,13 @@ Glossary the :term:`global interpreter lock` which allows only one thread to execute Python bytecode at a time. See :pep:`703`. + free-threaded build + + A build of :term:`CPython` that supports :term:`free threading`, + configured using the :option:`--disable-gil` option before compilation. + + See :ref:`freethreading-python-howto`. + free variable Formally, as defined in the :ref:`language execution model `, a free variable is any variable used in a namespace which is not a local variable in that @@ -662,6 +716,14 @@ Glossary requires the GIL to be held in order to use it. This refers to having an :term:`attached thread state`. + global state + Data that is accessible throughout a program, such as module-level + variables, class variables, or C static variables in :term:`extension modules + `. In multi-threaded programs, global state shared + between threads typically requires synchronization to avoid + :term:`race conditions ` and + :term:`data races `. + hash-based pyc A bytecode cache file that uses the hash rather than the last-modified time of the corresponding source file to determine its validity. See @@ -706,7 +768,9 @@ Glossary tuples. Such an object cannot be altered. A new object has to be created if a different value has to be stored. They play an important role in places where a constant hash value is needed, for example as a key - in a dictionary. + in a dictionary. Immutable objects are inherently :term:`thread-safe` + because their state cannot be modified after creation, eliminating concerns + about improperly synchronized :term:`concurrent modification`. import path A list of locations (or :term:`path entries `) that are @@ -723,6 +787,19 @@ Glossary An object that both finds and loads a module; both a :term:`finder` and :term:`loader` object. + index + A numeric value that represents the position of an element in + a :term:`sequence`. + + In Python, indexing starts at zero. + For example, ``things[0]`` names the *first* element of ``things``; + ``things[1]`` names the second one. + + In some contexts, Python allows negative indexes for counting from the + end of a sequence, and indexing using :term:`slices `. + + See also :term:`subscript`. + interactive Python has an interactive interpreter which means you can enter statements and expressions at the interpreter prompt, immediately @@ -796,9 +873,13 @@ Glossary CPython does not consistently apply the requirement that an iterator define :meth:`~iterator.__iter__`. - And also please note that the free-threading CPython does not guarantee - the thread-safety of iterator operations. + And also please note that :term:`free-threaded ` + CPython does not guarantee :term:`thread-safe` behavior of iterator + operations. + key + A value that identifies an entry in a :term:`mapping`. + See also :term:`subscript`. key function A key function or collation function is a callable that returns a value @@ -813,7 +894,7 @@ Glossary :func:`itertools.groupby`. There are several ways to create a key function. For example. the - :meth:`str.lower` method can serve as a key function for case insensitive + :meth:`str.casefold` method can serve as a key function for case insensitive sorts. Alternatively, a key function can be built from a :keyword:`lambda` expression such as ``lambda r: (r[0], r[2])``. Also, :func:`operator.attrgetter`, :func:`operator.itemgetter`, and @@ -835,10 +916,11 @@ Glossary :keyword:`if` statements. In a multi-threaded environment, the LBYL approach can risk introducing a - race condition between "the looking" and "the leaping". For example, the - code, ``if key in mapping: return mapping[key]`` can fail if another + :term:`race condition` between "the looking" and "the leaping". For example, + the code, ``if key in mapping: return mapping[key]`` can fail if another thread removes *key* from *mapping* after the test, but before the lookup. - This issue can be solved with locks or by using the EAFP approach. + This issue can be solved with :term:`locks ` or by using the + :term:`EAFP` approach. See also :term:`thread-safe`. lexical analyzer @@ -857,6 +939,29 @@ Glossary clause is optional. If omitted, all elements in ``range(256)`` are processed. + lock + A :term:`synchronization primitive` that allows only one thread at a + time to access a shared resource. A thread must acquire a lock before + accessing the protected resource and release it afterward. If a thread + attempts to acquire a lock that is already held by another thread, it + will block until the lock becomes available. Python's :mod:`threading` + module provides :class:`~threading.Lock` (a basic lock) and + :class:`~threading.RLock` (a :term:`reentrant` lock). Locks are used + to prevent :term:`race conditions ` and ensure + :term:`thread-safe` access to shared data. Alternative design patterns + to locks exist such as queues, producer/consumer patterns, and + thread-local state. See also :term:`deadlock`, and :term:`reentrant`. + + lock-free + An operation that does not acquire any :term:`lock` and uses atomic CPU + instructions to ensure correctness. Lock-free operations can execute + concurrently without blocking each other and cannot be blocked by + operations that hold locks. In :term:`free-threaded ` + Python, built-in types like :class:`dict` and :class:`list` provide + lock-free read operations, which means other threads may observe + intermediate states during multi-step modifications even when those + modifications hold the :term:`per-object lock`. + loader An object that loads a module. It must define the :meth:`!exec_module` and :meth:`!create_module` methods @@ -942,8 +1047,11 @@ Glossary See :term:`method resolution order`. mutable - Mutable objects can change their value but keep their :func:`id`. See - also :term:`immutable`. + An :term:`object` with state that is allowed to change during the course + of the program. In multi-threaded programs, mutable objects that are + shared between threads require careful synchronization to avoid + :term:`race conditions `. See also :term:`immutable`, + :term:`thread-safe`, and :term:`concurrent modification`. named tuple The term "named tuple" applies to any type or class that inherits from @@ -995,6 +1103,13 @@ Glossary See also :term:`module`. + native code + Code that is compiled to machine instructions and runs directly on the + processor, as opposed to code that is interpreted or runs in a virtual + machine. In the context of Python, native code typically refers to + C, C++, Rust or Fortran code in :term:`extension modules ` + that can be called from Python. See also :term:`extension module`. + nested scope The ability to refer to a variable in an enclosing definition. For instance, a function defined inside another function can refer to @@ -1011,6 +1126,15 @@ Glossary properties, :meth:`~object.__getattribute__`, class methods, and static methods. + non-deterministic + Behavior where the outcome of a program can vary between executions with + the same inputs. In multi-threaded programs, non-deterministic behavior + often results from :term:`race conditions ` where the + relative timing or interleaving of threads affects the result. + Proper synchronization using :term:`locks ` and other + :term:`synchronization primitives ` helps + ensure deterministic behavior. + object Any data with state (attributes or value) and defined behavior (methods). Also the ultimate base class of any :term:`new-style @@ -1025,6 +1149,15 @@ Glossary applied to all scopes, only those relying on a known set of local and nonlocal variable names are restricted to optimized scopes. + optional module + An :term:`extension module` that is part of the :term:`standard library`, + but may be absent in some builds of :term:`CPython`, + usually due to missing third-party libraries or because the module + is not available for a given platform. + + See :ref:`optional-module-requirements` for a list of optional modules + that require third-party libraries. + package A Python :term:`module` which can contain submodules or recursively, subpackages. Technically, a package is a Python module with a @@ -1032,6 +1165,16 @@ Glossary See also :term:`regular package` and :term:`namespace package`. + parallelism + Executing multiple operations at the same time (e.g. on multiple CPU + cores). In Python builds with the + :term:`global interpreter lock (GIL) `, only one + thread runs Python bytecode at a time, so taking advantage of multiple + CPU cores typically involves multiple processes + (e.g. :mod:`multiprocessing`) or native extensions that release the GIL. + In :term:`free-threaded ` Python, multiple Python threads + can run Python code simultaneously on different cores. + parameter A named entity in a :term:`function` (or method) definition that specifies an :term:`argument` (or in some cases, arguments) that the @@ -1085,6 +1228,16 @@ Glossary `, the :class:`inspect.Parameter` class, the :ref:`function` section, and :pep:`362`. + per-object lock + A :term:`lock` associated with an individual object instance rather than + a global lock shared across all objects. In :term:`free-threaded + ` Python, built-in types like :class:`dict` and + :class:`list` use per-object locks to allow concurrent operations on + different objects while serializing operations on the same object. + Operations that hold the per-object lock prevent other locking operations + on the same object from proceeding, but do not block :term:`lock-free` + operations. + path entry A single location on the :term:`import path` which the :term:`path based finder` consults to find modules for importing. @@ -1206,6 +1359,18 @@ Glossary >>> email.mime.text.__name__ 'email.mime.text' + race condition + A condition of a program where the behavior + depends on the relative timing or ordering of events, particularly in + multi-threaded programs. Race conditions can lead to + :term:`non-deterministic` behavior and bugs that are difficult to + reproduce. A :term:`data race` is a specific type of race condition + involving unsynchronized access to shared memory. The :term:`LBYL` + coding style is particularly susceptible to race conditions in + multi-threaded code. Using :term:`locks ` and other + :term:`synchronization primitives ` + helps prevent race conditions. + reference count The number of references to an object. When the reference count of an object drops to zero, it is deallocated. Some objects are @@ -1227,6 +1392,25 @@ Glossary See also :term:`namespace package`. + reentrant + A property of a function or :term:`lock` that allows it to be called or + acquired multiple times by the same thread without causing errors or a + :term:`deadlock`. + + For functions, reentrancy means the function can be safely called again + before a previous invocation has completed, which is important when + functions may be called recursively or from signal handlers. Thread-unsafe + functions may be :term:`non-deterministic` if they're called reentrantly in a + multithreaded program. + + For locks, Python's :class:`threading.RLock` (reentrant lock) is + reentrant, meaning a thread that already holds the lock can acquire it + again without blocking. In contrast, :class:`threading.Lock` is not + reentrant - attempting to acquire it twice from the same thread will cause + a deadlock. + + See also :term:`lock` and :term:`deadlock`. + REPL An acronym for the "read–eval–print loop", another name for the :term:`interactive` interpreter shell. @@ -1270,10 +1454,11 @@ Glossary chosen based on the type of a single argument. slice - An object usually containing a portion of a :term:`sequence`. A slice is - created using the subscript notation, ``[]`` with colons between numbers - when several are given, such as in ``variable_name[1:3:5]``. The bracket - (subscript) notation uses :class:`slice` objects internally. + An object of type :class:`slice`, used to describe a portion of + a :term:`sequence`. + A slice object is created when using the :ref:`slicing ` form + of :ref:`subscript notation `, with colons inside square + brackets, such as in ``variable_name[1:3:5]``. soft deprecated A soft deprecated API should not be used in new code, @@ -1331,6 +1516,26 @@ Glossary See also :term:`borrowed reference`. + subscript + The expression in square brackets of a + :ref:`subscription expression `, for example, + the ``3`` in ``items[3]``. + Usually used to select an element of a container. + Also called a :term:`key` when subscripting a :term:`mapping`, + or an :term:`index` when subscripting a :term:`sequence`. + + synchronization primitive + A basic building block for coordinating (synchronizing) the execution of + multiple threads to ensure :term:`thread-safe` access to shared resources. + Python's :mod:`threading` module provides several synchronization primitives + including :class:`~threading.Lock`, :class:`~threading.RLock`, + :class:`~threading.Semaphore`, :class:`~threading.Condition`, + :class:`~threading.Event`, and :class:`~threading.Barrier`. Additionally, + the :mod:`queue` module provides multi-producer, multi-consumer queues + that are especially useful in multithreaded programs. These + primitives help prevent :term:`race conditions ` and + coordinate thread execution. See also :term:`lock`. + t-string t-strings String literals prefixed with ``t`` or ``T`` are commonly called @@ -1383,6 +1588,19 @@ Glossary See :ref:`Thread State and the Global Interpreter Lock ` for more information. + thread-safe + A module, function, or class that behaves correctly when used by multiple + threads concurrently. Thread-safe code uses appropriate + :term:`synchronization primitives ` like + :term:`locks ` to protect shared mutable state, or is designed + to avoid shared mutable state entirely. In the + :term:`free-threaded ` build, built-in types like + :class:`dict`, :class:`list`, and :class:`set` use internal locking + to make many operations thread-safe, although thread safety is not + necessarily guaranteed. Code that is not thread-safe may experience + :term:`race conditions ` and :term:`data races ` + when used in multi-threaded programs. + token A small unit of source code, generated by the diff --git a/Doc/howto/a-conceptual-overview-of-asyncio.rst b/Doc/howto/a-conceptual-overview-of-asyncio.rst index d68f7cc6921..3adfedbf410 100644 --- a/Doc/howto/a-conceptual-overview-of-asyncio.rst +++ b/Doc/howto/a-conceptual-overview-of-asyncio.rst @@ -1,7 +1,7 @@ .. _a-conceptual-overview-of-asyncio: **************************************** -A Conceptual Overview of :mod:`!asyncio` +A conceptual overview of :mod:`!asyncio` **************************************** This :ref:`HOWTO ` article seeks to help you build a sturdy mental @@ -9,12 +9,11 @@ model of how :mod:`asyncio` fundamentally works, helping you understand the how and why behind the recommended patterns. You might be curious about some key :mod:`!asyncio` concepts. -You'll be comfortably able to answer these questions by the end of this -article: +By the end of this article, you'll be able to comfortably answer these questions: - What's happening behind the scenes when an object is awaited? - How does :mod:`!asyncio` differentiate between a task which doesn't need - CPU-time (such as a network request or file read) as opposed to a task that + CPU time (such as a network request or file read) as opposed to a task that does (such as computing n-factorial)? - How to write an asynchronous variant of an operation, such as an async sleep or database request. @@ -35,18 +34,18 @@ A conceptual overview part 1: the high-level -------------------------------------------- In part 1, we'll cover the main, high-level building blocks of :mod:`!asyncio`: -the event loop, coroutine functions, coroutine objects, tasks and ``await``. +the event loop, coroutine functions, coroutine objects, tasks, and ``await``. ========== -Event Loop +Event loop ========== Everything in :mod:`!asyncio` happens relative to the event loop. -It's the star of the show. +It's the star of the show, but prefers to work behind the scenes, managing +and coordinating resources. It's like an orchestra conductor. -It's behind the scenes managing resources. Some power is explicitly granted to it, but a lot of its ability to get things -done comes from the respect and cooperation of its worker bees. +done comes from the respect and cooperation of its band members. In more technical terms, the event loop contains a collection of jobs to be run. Some jobs are added directly by you, and some indirectly by :mod:`!asyncio`. @@ -56,11 +55,11 @@ Once it pauses or completes, it returns control to the event loop. The event loop will then select another job from its pool and invoke it. You can *roughly* think of the collection of jobs as a queue: jobs are added and then processed one at a time, generally (but not always) in order. -This process repeats indefinitely with the event loop cycling endlessly +This process repeats indefinitely, with the event loop cycling endlessly onwards. If there are no more jobs pending execution, the event loop is smart enough to rest and avoid needlessly wasting CPU cycles, and will come back when there's -more work to be done. +more work to be done, such as when I/O operations complete or timers expire. Effective execution relies on jobs sharing well and cooperating; a greedy job could hog control and leave the other jobs to starve, rendering the overall @@ -171,14 +170,17 @@ Roughly speaking, :ref:`tasks ` are coroutines (not coroutine functions) tied to an event loop. A task also maintains a list of callback functions whose importance will become clear in a moment when we discuss :keyword:`await`. -The recommended way to create tasks is via :func:`asyncio.create_task`. Creating a task automatically schedules it for execution (by adding a callback to run it in the event loop's to-do list, that is, collection of jobs). +The recommended way to create tasks is via :func:`asyncio.create_task`. -Since there's only one event loop (in each thread), :mod:`!asyncio` takes care of -associating the task with the event loop for you. As such, there's no need -to specify the event loop. +:mod:`!asyncio` automatically associates tasks with the event loop for you. +This automatic association was purposely designed into :mod:`!asyncio` for +the sake of simplicity. +Without it, you'd have to keep track of the event loop object and pass it to +any coroutine function that wants to create tasks, adding redundant clutter +to your code. :: @@ -251,6 +253,10 @@ different ways:: In a crucial way, the behavior of ``await`` depends on the type of object being awaited. +^^^^^^^^^^^^^^ +Awaiting tasks +^^^^^^^^^^^^^^ + Awaiting a task will cede control from the current task or coroutine to the event loop. In the process of relinquishing control, a few important things happen. @@ -276,12 +282,16 @@ in this case, a call to resume ``plant_a_tree()``. Generally speaking, when the awaited task finishes (``dig_the_hole_task``), the original task or coroutine (``plant_a_tree()``) is added back to the event -loops to-do list to be resumed. +loop's to-do list to be resumed. This is a basic, yet reliable mental model. In practice, the control handoffs are slightly more complex, but not by much. In part 2, we'll walk through the details that make this possible. +^^^^^^^^^^^^^^^^^^^ +Awaiting coroutines +^^^^^^^^^^^^^^^^^^^ + **Unlike tasks, awaiting a coroutine does not hand control back to the event loop!** Wrapping a coroutine in a task first, then awaiting that would cede @@ -310,7 +320,7 @@ Consider this program:: The first statement in the coroutine ``main()`` creates ``task_b`` and schedules it for execution via the event loop. Then, ``coro_a()`` is repeatedly awaited. Control never cedes to the -event loop which is why we see the output of all three ``coro_a()`` +event loop, which is why we see the output of all three ``coro_a()`` invocations before ``coro_b()``'s output: .. code-block:: none @@ -338,8 +348,8 @@ This behavior of ``await coroutine`` can trip a lot of people up! That example highlights how using only ``await coroutine`` could unintentionally hog control from other tasks and effectively stall the event loop. -:func:`asyncio.run` can help you detect such occurences via the -``debug=True`` flag which accordingly enables +:func:`asyncio.run` can help you detect such occurrences via the +``debug=True`` flag, which enables :ref:`debug mode `. Among other things, it will log any coroutines that monopolize execution for 100ms or longer. @@ -348,8 +358,10 @@ The design intentionally trades off some conceptual clarity around usage of ``await`` for improved performance. Each time a task is awaited, control needs to be passed all the way up the call stack to the event loop. -That might sound minor, but in a large program with many ``await``'s and a deep -callstack that overhead can add up to a meaningful performance drag. +Then, the event loop needs to manage its internal state and work through +its processing logic to resume the next job. +That might sound minor, but in a large program with many ``await``\ s, that +overhead can add up to a non-negligible performance drag. ------------------------------------------------ A conceptual overview part 2: the nuts and bolts @@ -365,14 +377,15 @@ and how to make your own asynchronous operators. The inner workings of coroutines ================================ -:mod:`!asyncio` leverages four components to pass around control. +:mod:`!asyncio` leverages four components of Python to pass +around control. :meth:`coroutine.send(arg) ` is the method used to start or resume a coroutine. If the coroutine was paused and is now being resumed, the argument ``arg`` will be sent in as the return value of the ``yield`` statement which originally paused it. -If the coroutine is being used for the first time (as opposed to being resumed) +If the coroutine is being used for the first time (as opposed to being resumed), ``arg`` must be ``None``. .. code-block:: @@ -403,14 +416,14 @@ If the coroutine is being used for the first time (as opposed to being resumed) returned_value = e.value print(f"Coroutine main() finished and provided value: {returned_value}.") -:ref:`yield `, like usual, pauses execution and returns control +:ref:`yield `, as usual, pauses execution and returns control to the caller. In the example above, the ``yield``, on line 3, is called by ``... = await rock`` on line 11. More broadly speaking, ``await`` calls the :meth:`~object.__await__` method of the given object. ``await`` also does one more very special thing: it propagates (or "passes -along") any ``yield``\ s it receives up the call-chain. +along") any ``yield``\ s it receives up the call chain. In this case, that's back to ``... = coroutine.send(None)`` on line 16. The coroutine is resumed via the ``coroutine.send(42)`` call on line 21. @@ -449,9 +462,9 @@ That might sound odd to you. You might be thinking: That causes the error: ``SyntaxError: yield from not allowed in a coroutine.`` This was intentionally designed for the sake of simplicity -- mandating only one way of using coroutines. + Despite that, ``yield from`` and ``await`` effectively do the same thing. Initially ``yield`` was barred as well, but was re-accepted to allow for async generators. - Despite that, ``yield from`` and ``await`` effectively do the same thing. ======= Futures @@ -462,12 +475,12 @@ computation's status and result. The term is a nod to the idea of something still to come or not yet happened, and the object is a way to keep an eye on that something. -A future has a few important attributes. One is its state which can be either -"pending", "cancelled" or "done". +A future has a few important attributes. One is its state, which can be either +"pending", "cancelled", or "done". Another is its result, which is set when the state transitions to done. Unlike a coroutine, a future does not represent the actual computation to be done; instead, it represents the status and result of that computation, kind of -like a status light (red, yellow or green) or indicator. +like a status light (red, yellow, or green) or indicator. :class:`asyncio.Task` subclasses :class:`asyncio.Future` in order to gain these various capabilities. @@ -490,8 +503,8 @@ We'll go through an example of how you could leverage a future to create your own variant of asynchronous sleep (``async_sleep``) which mimics :func:`asyncio.sleep`. -This snippet registers a few tasks with the event loop and then awaits a -coroutine wrapped in a task: ``async_sleep(3)``. +This snippet registers a few tasks with the event loop and then awaits the task +created by ``asyncio.create_task``, which wraps the ``async_sleep(3)`` coroutine. We want that task to finish only after three seconds have elapsed, but without preventing other tasks from running. @@ -540,8 +553,8 @@ will monitor how much time has elapsed and, accordingly, call # Block until the future is marked as done. await future -Below, we'll use a rather bare object, ``YieldToEventLoop()``, to ``yield`` -from ``__await__`` in order to cede control to the event loop. +Below, we use a rather bare ``YieldToEventLoop()`` object to ``yield`` +from its ``__await__`` method, ceding control to the event loop. This is effectively the same as calling ``asyncio.sleep(0)``, but this approach offers more clarity, not to mention it's somewhat cheating to use ``asyncio.sleep`` when showcasing how to implement it! @@ -552,13 +565,13 @@ The ``watcher_task``, which runs the coroutine ``_sleep_watcher(...)``, will be invoked once per full cycle of the event loop. On each resumption, it'll check the time and if not enough has elapsed, then it'll pause once again and hand control back to the event loop. -Eventually, enough time will have elapsed, and ``_sleep_watcher(...)`` will -mark the future as done, and then itself finish too by breaking out of the +Once enough time has elapsed, ``_sleep_watcher(...)`` +marks the future as done and completes by exiting its infinite ``while`` loop. Given this helper task is only invoked once per cycle of the event loop, you'd be correct to note that this asynchronous sleep will sleep *at least* three seconds, rather than exactly three seconds. -Note this is also of true of ``asyncio.sleep``. +Note this is also true of ``asyncio.sleep``. :: @@ -601,6 +614,6 @@ For reference, you could implement it without futures, like so:: else: await YieldToEventLoop() -But, that's all for now. Hopefully you're ready to more confidently dive into +But that's all for now. Hopefully you're ready to more confidently dive into some async programming or check out advanced topics in the :mod:`rest of the documentation `. diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 9d5a9ac8b71..a7a68281860 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -594,7 +594,7 @@ a pure Python equivalent: def object_getattribute(obj, name): "Emulate PyObject_GenericGetAttr() in Objects/object.c" - null = object() + null = sentinel('null') objtype = type(obj) cls_var = find_name_in_mro(objtype, name, null) descr_get = getattr(type(cls_var), '__get__', null) @@ -1635,12 +1635,12 @@ by member descriptors: .. testcode:: - null = object() + null = sentinel('null') class Member: def __init__(self, name, clsname, offset): - 'Emulate PyMemberDef in Include/structmember.h' + 'Emulate PyMemberDef in Include/descrobject.h' # Also see descr_new() in Objects/descrobject.c self.name = name self.clsname = clsname diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 7713aede6d5..2fe5814bb04 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -105,8 +105,8 @@ The complete :class:`!Weekday` enum now looks like this:: Now we can find out what today is! Observe:: - >>> from datetime import date - >>> Weekday.from_date(date.today()) # doctest: +SKIP + >>> import datetime as dt + >>> Weekday.from_date(dt.date.today()) # doctest: +SKIP Of course, if you're reading this on some other day, you'll see that day instead. @@ -371,7 +371,7 @@ Equality comparisons are defined though:: >>> Color.BLUE == Color.BLUE True -Comparisons against non-enumeration values will always compare not equal +Equality comparisons against non-enumeration values will always return ``False`` (again, :class:`IntEnum` was explicitly designed to behave differently, see below):: @@ -965,75 +965,16 @@ want one of them to be the value:: Finer Points -^^^^^^^^^^^^ +------------ -Supported ``__dunder__`` names -"""""""""""""""""""""""""""""" +Supported ``__dunder__`` and ``_sunder_`` names +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:attr:`~enum.EnumType.__members__` is a read-only ordered mapping of ``member_name``:``member`` -items. It is only available on the class. - -:meth:`~object.__new__`, if specified, must create and return the enum members; it is -also a very good idea to set the member's :attr:`~Enum._value_` appropriately. Once -all the members are created it is no longer used. - - -Supported ``_sunder_`` names -"""""""""""""""""""""""""""" - -- :attr:`~Enum._name_` -- name of the member -- :attr:`~Enum._value_` -- value of the member; can be set in ``__new__`` -- :meth:`~Enum._missing_` -- a lookup function used when a value is not found; - may be overridden -- :attr:`~Enum._ignore_` -- a list of names, either as a :class:`list` or a - :class:`str`, that will not be transformed into members, and will be removed - from the final class -- :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for - an enum member; may be overridden -- :meth:`~Enum._add_alias_` -- adds a new name as an alias to an existing - member. -- :meth:`~Enum._add_value_alias_` -- adds a new value as an alias to an - existing member. See `MultiValueEnum`_ for an example. - - .. note:: - - For standard :class:`Enum` classes the next value chosen is the highest - value seen incremented by one. - - For :class:`Flag` classes the next value chosen will be the next highest - power-of-two. - - .. versionchanged:: 3.13 - Prior versions would use the last seen value instead of the highest value. - -.. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` -.. versionadded:: 3.7 ``_ignore_`` -.. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_`` - -To help keep Python 2 / Python 3 code in sync an :attr:`~Enum._order_` attribute can -be provided. It will be checked against the actual order of the enumeration -and raise an error if the two do not match:: - - >>> class Color(Enum): - ... _order_ = 'RED GREEN BLUE' - ... RED = 1 - ... BLUE = 3 - ... GREEN = 2 - ... - Traceback (most recent call last): - ... - TypeError: member order does not match _order_: - ['RED', 'BLUE', 'GREEN'] - ['RED', 'GREEN', 'BLUE'] - -.. note:: - - In Python 2 code the :attr:`~Enum._order_` attribute is necessary as definition - order is lost before it can be recorded. +The supported ``__dunder__`` and ``_sunder_`` names can be found in the :ref:`Enum API documentation `. _Private__names -""""""""""""""" +^^^^^^^^^^^^^^^ :ref:`Private names ` are not converted to enum members, but remain normal attributes. @@ -1042,7 +983,7 @@ but remain normal attributes. ``Enum`` member type -"""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^ Enum members are instances of their enum class, and are normally accessed as ``EnumClass.member``. In certain situations, such as writing custom enum @@ -1055,7 +996,7 @@ recommended. Creating members that are mixed with other data types -""""""""""""""""""""""""""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When subclassing other data types, such as :class:`int` or :class:`str`, with an :class:`Enum`, all values after the ``=`` are passed to that data type's @@ -1069,7 +1010,7 @@ constructor. For example:: Boolean value of ``Enum`` classes and members -""""""""""""""""""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Enum classes that are mixed with non-:class:`Enum` types (such as :class:`int`, :class:`str`, etc.) are evaluated according to the mixed-in @@ -1084,7 +1025,7 @@ Plain :class:`Enum` classes always evaluate as :data:`True`. ``Enum`` classes with methods -""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you give your enum subclass extra methods, like the `Planet`_ class below, those methods will show up in a :func:`dir` of the member, @@ -1097,7 +1038,7 @@ but not of the class:: Combining members of ``Flag`` -""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Iterating over a combination of :class:`Flag` members will only return the members that are comprised of a single bit:: @@ -1117,7 +1058,7 @@ are comprised of a single bit:: ``Flag`` and ``IntFlag`` minutia -"""""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Using the following snippet for our examples:: @@ -1478,6 +1419,7 @@ alias:: behaviors as well as disallowing aliases. If the only desired change is disallowing aliases, the :func:`unique` decorator can be used instead. +.. _multi-value-enum: MultiValueEnum ^^^^^^^^^^^^^^^^^ @@ -1538,8 +1480,8 @@ TimePeriod An example to show the :attr:`~Enum._ignore_` attribute in use:: - >>> from datetime import timedelta - >>> class Period(timedelta, Enum): + >>> import datetime as dt + >>> class Period(dt.timedelta, Enum): ... "different lengths of time" ... _ignore_ = 'Period i' ... Period = vars() diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 577e283bb9c..b21ed1c8f37 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -45,9 +45,12 @@ single-phase initialization. Multi-Phase Initialization .......................... -Extensions that use multi-phase initialization (i.e., -:c:func:`PyModuleDef_Init`) should add a :c:data:`Py_mod_gil` slot in the -module definition. If your extension supports older versions of CPython, +Extensions that use :ref:`multi-phase initialization ` +(functions like :c:func:`PyModuleDef_Init`, +:c:func:`PyModExport_* ` export hook, +:c:func:`PyModule_FromSlotsAndSpec`) should add a +:c:data:`Py_mod_gil` slot in the module definition. +If your extension supports older versions of CPython, you should guard the slot with a :c:data:`PY_VERSION_HEX` check. :: @@ -60,18 +63,12 @@ you should guard the slot with a :c:data:`PY_VERSION_HEX` check. {0, NULL} }; - static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - .m_slots = module_slots, - ... - }; - Single-Phase Initialization ........................... -Extensions that use single-phase initialization (i.e., -:c:func:`PyModule_Create`) should call :c:func:`PyUnstable_Module_SetGIL` to +Extensions that use legacy :ref:`single-phase initialization ` +(that is, :c:func:`PyModule_Create`) should call :c:func:`PyUnstable_Module_SetGIL` to indicate that they support running with the GIL disabled. The function is only defined in the free-threaded build, so you should guard the call with ``#ifdef Py_GIL_DISABLED`` to avoid compilation errors in the regular build. @@ -173,9 +170,9 @@ that return :term:`strong references `. +-----------------------------------+-----------------------------------+ | :c:func:`PyDict_Next` | none (see :ref:`PyDict_Next`) | +-----------------------------------+-----------------------------------+ -| :c:func:`PyWeakref_GetObject` | :c:func:`PyWeakref_GetRef` | +| :c:func:`!PyWeakref_GetObject` | :c:func:`PyWeakref_GetRef` | +-----------------------------------+-----------------------------------+ -| :c:func:`PyWeakref_GET_OBJECT` | :c:func:`PyWeakref_GetRef` | +| :c:func:`!PyWeakref_GET_OBJECT` | :c:func:`PyWeakref_GetRef` | +-----------------------------------+-----------------------------------+ | :c:func:`PyImport_AddModule` | :c:func:`PyImport_AddModuleRef` | +-----------------------------------+-----------------------------------+ @@ -203,7 +200,7 @@ Memory Allocation APIs Python's memory management C API provides functions in three different :ref:`allocation domains `: "raw", "mem", and "object". For thread-safety, the free-threaded build requires that only Python objects -are allocated using the object domain, and that all Python object are +are allocated using the object domain, and that all Python objects are allocated using that domain. This differs from the prior Python versions, where this was only a best practice and not a hard requirement. @@ -344,12 +341,12 @@ This means you cannot rely on nested critical sections to lock multiple objects at once, as the inner critical section may suspend the outer ones. Instead, use :c:macro:`Py_BEGIN_CRITICAL_SECTION2` to lock two objects simultaneously. -Note that the locks described above are only :c:type:`!PyMutex` based locks. +Note that the locks described above are only :c:type:`PyMutex` based locks. The critical section implementation does not know about or affect other locking mechanisms that might be in use, like POSIX mutexes. Also note that while -blocking on any :c:type:`!PyMutex` causes the critical sections to be +blocking on any :c:type:`PyMutex` causes the critical sections to be suspended, only the mutexes that are part of the critical sections are -released. If :c:type:`!PyMutex` is used without a critical section, it will +released. If :c:type:`PyMutex` is used without a critical section, it will not be released and therefore does not get the same deadlock avoidance. Important Considerations @@ -387,6 +384,30 @@ Important Considerations internal extension state, standard mutexes or other synchronization primitives might be more appropriate. +.. _per-object-locks: + +Per-Object Locks (``ob_mutex``) +............................... + +In the free-threaded build, each Python object contains a :c:member:`~PyObject.ob_mutex` +field of type :c:type:`PyMutex`. This mutex is **reserved for use by the +critical section API** (:c:macro:`Py_BEGIN_CRITICAL_SECTION` / +:c:macro:`Py_END_CRITICAL_SECTION`). + +.. warning:: + + Do **not** lock ``ob_mutex`` directly with ``PyMutex_Lock(&obj->ob_mutex)``. + Mixing direct ``PyMutex_Lock`` calls with the critical section API on the + same mutex can cause deadlocks. + +Even if your own code never uses critical sections on a particular object type, +**CPython internals may use the critical section API on any Python object**. + +If your extension type needs its own lock, add a separate :c:type:`PyMutex` +field (or another synchronization primitive) to your object struct. +:c:type:`PyMutex` is very lightweight, so there is negligible cost to having +an additional one. + Building Extensions for the Free-Threaded Build =============================================== @@ -395,10 +416,9 @@ C API extensions need to be built specifically for the free-threaded build. The wheels, shared libraries, and binaries are indicated by a ``t`` suffix. * `pypa/manylinux `_ supports the - free-threaded build, with the ``t`` suffix, such as ``python3.13t``. -* `pypa/cibuildwheel `_ supports the - free-threaded build if you set - `CIBW_ENABLE to cpython-freethreading `_. + free-threaded build, with the ``t`` suffix, such as ``python3.14t``. +* `pypa/cibuildwheel `_ supports + building wheels for the free-threaded build of Python 3.14 and newer. Limited C API and Stable ABI ............................ diff --git a/Doc/howto/free-threading-python.rst b/Doc/howto/free-threading-python.rst index 24069617c47..380c2be0495 100644 --- a/Doc/howto/free-threading-python.rst +++ b/Doc/howto/free-threading-python.rst @@ -11,9 +11,7 @@ available processing power by running threads in parallel on available CPU cores While not all software will benefit from this automatically, programs designed with threading in mind will run faster on multi-core hardware. -The free-threaded mode is working and continues to be improved, but -there is some additional overhead in single-threaded workloads compared -to the regular build. Additionally, third-party packages, in particular ones +Some third-party packages, in particular ones with an :term:`extension module`, may not be ready for use in a free-threaded build, and will re-enable the :term:`GIL`. @@ -101,60 +99,42 @@ This section describes known limitations of the free-threaded CPython build. Immortalization --------------- -The free-threaded build of the 3.13 release makes some objects :term:`immortal`. +In the free-threaded build, some objects are :term:`immortal`. Immortal objects are not deallocated and have reference counts that are never modified. This is done to avoid reference count contention that would prevent efficient multi-threaded scaling. -An object will be made immortal when a new thread is started for the first time -after the main thread is running. The following objects are immortalized: +As of the 3.14 release, immortalization is limited to: -* :ref:`function ` objects declared at the module level -* :ref:`method ` descriptors -* :ref:`code ` objects -* :term:`module` objects and their dictionaries -* :ref:`classes ` (type objects) - -Because immortal objects are never deallocated, applications that create many -objects of these types may see increased memory usage. This is expected to be -addressed in the 3.14 release. - -Additionally, numeric and string literals in the code as well as strings -returned by :func:`sys.intern` are also immortalized. This behavior is -expected to remain in the 3.14 free-threaded build. +* Code constants: numeric literals, string literals, and tuple literals + composed of other constants. +* Strings interned by :func:`sys.intern`. Frame objects ------------- -It is not safe to access :ref:`frame ` objects from other -threads and doing so may cause your program to crash . This means that -:func:`sys._current_frames` is generally not safe to use in a free-threaded -build. Functions like :func:`inspect.currentframe` and :func:`sys._getframe` -are generally safe as long as the resulting frame object is not passed to -another thread. +It is not safe to access :attr:`frame.f_locals` from a :ref:`frame ` +object if that frame is currently executing in another thread, and doing so may +crash the interpreter. + Iterators --------- -Sharing the same iterator object between multiple threads is generally not -safe and threads may see duplicate or missing elements when iterating or crash -the interpreter. +It is generally not thread-safe to access the same iterator object from +multiple threads concurrently, and threads may see duplicate or missing +elements. Single-threaded performance --------------------------- The free-threaded build has additional overhead when executing Python code -compared to the default GIL-enabled build. In 3.13, this overhead is about -40% on the `pyperformance `_ suite. -Programs that spend most of their time in C extensions or I/O will see -less of an impact. The largest impact is because the specializing adaptive -interpreter (:pep:`659`) is disabled in the free-threaded build. We expect -to re-enable it in a thread-safe way in the 3.14 release. This overhead is -expected to be reduced in upcoming Python release. We are aiming for an -overhead of 10% or less on the pyperformance suite compared to the default -GIL-enabled build. +compared to the default GIL-enabled build. The amount of overhead depends +on the workload and hardware. On the pyperformance benchmark suite, the +average overhead ranges from about 1% on macOS aarch64 to 8% on x86-64 Linux +systems. Behavioral changes diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 053558e3890..552514063c9 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -4,7 +4,7 @@ Functional Programming HOWTO ******************************** -:Author: A. M. Kuchling +:Author: \A. M. Kuchling :Release: 0.32 In this document, we'll take a tour of Python's features suitable for diff --git a/Doc/howto/gdb_helpers.rst b/Doc/howto/gdb_helpers.rst index 98ce813ca4a..33d1fbf8cd9 100644 --- a/Doc/howto/gdb_helpers.rst +++ b/Doc/howto/gdb_helpers.rst @@ -136,7 +136,7 @@ enabled:: at Objects/unicodeobject.c:551 #7 0x0000000000440d94 in PyUnicodeUCS2_FromString (u=0x5c2b8d "__lltrace__") at Objects/unicodeobject.c:569 #8 0x0000000000584abd in PyDict_GetItemString (v= - {'Yuck': , '__builtins__': , '__file__': 'Lib/test/crashers/nasty_eq_vs_dict.py', '__package__': None, 'y': , 'dict': {0: 0, 1: 1, 2: 2, 3: 3}, '__cached__': None, '__name__': '__main__', 'z': , '__doc__': None}, key= + {'Yuck': , '__builtins__': , '__file__': 'Lib/test/crashers/nasty_eq_vs_dict.py', '__package__': None, 'y': , 'dict': {0: 0, 1: 1, 2: 2, 3: 3}, '__name__': '__main__', 'z': , '__doc__': None}, key= 0x5c2b8d "__lltrace__") at Objects/dictobject.c:2171 Notice how the dictionary argument to ``PyDict_GetItemString`` is displayed diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst index b3db1189e5d..06c1ae40da5 100644 --- a/Doc/howto/instrumentation.rst +++ b/Doc/howto/instrumentation.rst @@ -341,6 +341,84 @@ Available static markers .. versionadded:: 3.8 +C Entry Points +^^^^^^^^^^^^^^ + +To simplify triggering of DTrace markers, Python's C API comes with a number +of helper functions that mirror each static marker. On builds of Python without +DTrace enabled, these do nothing. + +In general, it is not necessary to call these yourself, as Python will do +it for you. + +.. list-table:: + :widths: 50 25 25 + :header-rows: 1 + + * * C API Function + * Static Marker + * Notes + * * .. c:function:: void PyDTrace_LINE(const char *arg0, const char *arg1, int arg2) + * :c:func:`!line` + * + * * .. c:function:: void PyDTrace_FUNCTION_ENTRY(const char *arg0, const char *arg1, int arg2) + * :c:func:`!function__entry` + * + * * .. c:function:: void PyDTrace_FUNCTION_RETURN(const char *arg0, const char *arg1, int arg2) + * :c:func:`!function__return` + * + * * .. c:function:: void PyDTrace_GC_START(int arg0) + * :c:func:`!gc__start` + * + * * .. c:function:: void PyDTrace_GC_DONE(Py_ssize_t arg0) + * :c:func:`!gc__done` + * + * * .. c:function:: void PyDTrace_INSTANCE_NEW_START(int arg0) + * :c:func:`!instance__new__start` + * Not used by Python + * * .. c:function:: void PyDTrace_INSTANCE_NEW_DONE(int arg0) + * :c:func:`!instance__new__done` + * Not used by Python + * * .. c:function:: void PyDTrace_INSTANCE_DELETE_START(int arg0) + * :c:func:`!instance__delete__start` + * Not used by Python + * * .. c:function:: void PyDTrace_INSTANCE_DELETE_DONE(int arg0) + * :c:func:`!instance__delete__done` + * Not used by Python + * * .. c:function:: void PyDTrace_IMPORT_FIND_LOAD_START(const char *arg0) + * :c:func:`!import__find__load__start` + * + * * .. c:function:: void PyDTrace_IMPORT_FIND_LOAD_DONE(const char *arg0, int arg1) + * :c:func:`!import__find__load__done` + * + * * .. c:function:: void PyDTrace_AUDIT(const char *arg0, void *arg1) + * :c:func:`!audit` + * + + +C Probing Checks +^^^^^^^^^^^^^^^^ + +.. c:function:: int PyDTrace_LINE_ENABLED(void) +.. c:function:: int PyDTrace_FUNCTION_ENTRY_ENABLED(void) +.. c:function:: int PyDTrace_FUNCTION_RETURN_ENABLED(void) +.. c:function:: int PyDTrace_GC_START_ENABLED(void) +.. c:function:: int PyDTrace_GC_DONE_ENABLED(void) +.. c:function:: int PyDTrace_INSTANCE_NEW_START_ENABLED(void) +.. c:function:: int PyDTrace_INSTANCE_NEW_DONE_ENABLED(void) +.. c:function:: int PyDTrace_INSTANCE_DELETE_START_ENABLED(void) +.. c:function:: int PyDTrace_INSTANCE_DELETE_DONE_ENABLED(void) +.. c:function:: int PyDTrace_IMPORT_FIND_LOAD_START_ENABLED(void) +.. c:function:: int PyDTrace_IMPORT_FIND_LOAD_DONE_ENABLED(void) +.. c:function:: int PyDTrace_AUDIT_ENABLED(void) + + All calls to ``PyDTrace`` functions must be guarded by a call to one + of these functions. This allows Python to minimize performance impact + when probing is disabled. + + On builds without DTrace enabled, these functions do nothing and return + ``0``. + SystemTap Tapsets ----------------- diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index 7da6dc8a397..6092c75f48f 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -353,7 +353,7 @@ garbage collection protocol. That is, heap types should: - Have the :c:macro:`Py_TPFLAGS_HAVE_GC` flag. -- Define a traverse function using ``Py_tp_traverse``, which +- Define a traverse function using :c:data:`Py_tp_traverse`, which visits the type (e.g. using ``Py_VISIT(Py_TYPE(self))``). Please refer to the documentation of diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 52537a91df5..0ee4c0086dd 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -229,7 +229,7 @@ messages should not. Here's how you can achieve this:: # tell the handler to use this format console.setFormatter(formatter) # add the handler to the root logger - logging.getLogger('').addHandler(console) + logging.getLogger().addHandler(console) # Now, we can log to the root logger, or any other logger. First the root... logging.info('Jackdaws love my big sphinx of quartz.') @@ -650,7 +650,7 @@ the receiving end. A simple way of doing this is attaching a import logging, logging.handlers - rootLogger = logging.getLogger('') + rootLogger = logging.getLogger() rootLogger.setLevel(logging.DEBUG) socketHandler = logging.handlers.SocketHandler('localhost', logging.handlers.DEFAULT_TCP_LOGGING_PORT) @@ -1549,10 +1549,10 @@ to this (remembering to first import :mod:`concurrent.futures`):: for i in range(10): executor.submit(worker_process, queue, worker_configurer) -Deploying Web applications using Gunicorn and uWSGI +Deploying web applications using Gunicorn and uWSGI ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -When deploying Web applications using `Gunicorn `_ or `uWSGI +When deploying web applications using `Gunicorn `_ or `uWSGI `_ (or similar), multiple worker processes are created to handle client requests. In such environments, avoid creating file-based handlers directly in your web application. Instead, use a @@ -1564,9 +1564,6 @@ process. This can be set up using a process management tool such as Supervisor - Using file rotation ------------------- -.. sectionauthor:: Doug Hellmann, Vinay Sajip (changes) -.. (see ) - Sometimes you want to let a log file grow to a certain size, then open a new file and log to that. You may want to keep a certain number of these files, and when that many files have been created, rotate the files so that the number of @@ -3619,7 +3616,6 @@ detailed information. .. code-block:: python3 - import datetime import logging import random import sys @@ -3854,7 +3850,7 @@ Logging to syslog with RFC5424 support Although :rfc:`5424` dates from 2009, most syslog servers are configured by default to use the older :rfc:`3164`, which hails from 2001. When ``logging`` was added to Python in 2003, it supported the earlier (and only existing) protocol at the time. Since -RFC5424 came out, as there has not been widespread deployment of it in syslog +RFC 5424 came out, as there has not been widespread deployment of it in syslog servers, the :class:`~logging.handlers.SysLogHandler` functionality has not been updated. @@ -3862,7 +3858,7 @@ RFC 5424 contains some useful features such as support for structured data, and need to be able to log to a syslog server with support for it, you can do so with a subclassed handler which looks something like this:: - import datetime + import datetime as dt import logging.handlers import re import socket @@ -3880,8 +3876,8 @@ subclassed handler which looks something like this:: def format(self, record): version = 1 - asctime = datetime.datetime.fromtimestamp(record.created).isoformat() - m = self.tz_offset.match(time.strftime('%z')) + asctime = dt.datetime.fromtimestamp(record.created).isoformat() + m = self.tz_offset.prefixmatch(time.strftime('%z')) has_offset = False if m and time.timezone: hrs, mins = m.groups() diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index b7225ff1c2c..454e2f4930e 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -28,7 +28,7 @@ When to use logging ^^^^^^^^^^^^^^^^^^^ You can access logging functionality by creating a logger via ``logger = -getLogger(__name__)``, and then calling the logger's :meth:`~Logger.debug`, +logging.getLogger(__name__)``, and then calling the logger's :meth:`~Logger.debug`, :meth:`~Logger.info`, :meth:`~Logger.warning`, :meth:`~Logger.error` and :meth:`~Logger.critical` methods. To determine when to use logging, and to see which logger methods to use when, see the table below. It states, for each of a diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index fc4772bbcca..653f28ddbab 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -217,8 +217,9 @@ Example, using the :mod:`sys` APIs in file :file:`example.py`: How to obtain the best results ------------------------------ -For best results, Python should be compiled with -``CFLAGS="-fno-omit-frame-pointer -mno-omit-leaf-frame-pointer"`` as this allows +For best results, keep frame pointers enabled. On supported GCC-compatible +toolchains, CPython builds itself with ``-fno-omit-frame-pointer`` and, when +available, ``-mno-omit-leaf-frame-pointer`` by default. These flags allow profilers to unwind using only the frame pointer and not on DWARF debug information. This is because as the code that is interposed to allow ``perf`` support is dynamically generated it doesn't have any DWARF debugging information diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index 7486a378dbb..6fc087c3f1c 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -1,7 +1,7 @@ .. _regex-howto: **************************** - Regular Expression HOWTO + Regular expression HOWTO **************************** :Author: A.M. Kuchling @@ -47,7 +47,7 @@ Python code to do the processing; while Python code will be slower than an elaborate regular expression, it will also probably be more understandable. -Simple Patterns +Simple patterns =============== We'll start by learning about the simplest possible regular expressions. Since @@ -59,7 +59,7 @@ expressions (deterministic and non-deterministic finite automata), you can refer to almost any textbook on writing compilers. -Matching Characters +Matching characters ------------------- Most letters and characters will simply match themselves. For example, the @@ -159,7 +159,7 @@ match even a newline. ``.`` is often used where you want to match "any character". -Repeating Things +Repeating things ---------------- Being able to match varying sets of characters is the first thing regular @@ -210,7 +210,7 @@ this RE against the string ``'abcbd'``. | | | ``[bcd]*`` is only matching | | | | ``bc``. | +------+-----------+---------------------------------+ -| 6 | ``abcb`` | Try ``b`` again. This time | +| 7 | ``abcb`` | Try ``b`` again. This time | | | | the character at the | | | | current position is ``'b'``, so | | | | it succeeds. | @@ -255,7 +255,7 @@ is equivalent to ``+``, and ``{0,1}`` is the same as ``?``. It's better to use to read. -Using Regular Expressions +Using regular expressions ========================= Now that we've looked at some simple regular expressions, how do we actually use @@ -264,7 +264,7 @@ expression engine, allowing you to compile REs into objects and then perform matches with them. -Compiling Regular Expressions +Compiling regular expressions ----------------------------- Regular expressions are compiled into pattern objects, which have @@ -295,7 +295,7 @@ disadvantage which is the topic of the next section. .. _the-backslash-plague: -The Backslash Plague +The backslash plague -------------------- As stated earlier, regular expressions use the backslash character (``'\'``) to @@ -335,7 +335,7 @@ expressions will often be written in Python code using this raw string notation. In addition, special escape sequences that are valid in regular expressions, but not valid as Python string literals, now result in a -:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`, +:exc:`SyntaxWarning` and will eventually become a :exc:`SyntaxError`, which means the sequences will be invalid if raw string notation or escaping the backslashes isn't used. @@ -351,7 +351,7 @@ the backslashes isn't used. +-------------------+------------------+ -Performing Matches +Performing matches ------------------ Once you have an object representing a compiled regular expression, what do you @@ -362,20 +362,21 @@ for a complete listing. +------------------+-----------------------------------------------+ | Method/Attribute | Purpose | +==================+===============================================+ -| ``match()`` | Determine if the RE matches at the beginning | -| | of the string. | -+------------------+-----------------------------------------------+ | ``search()`` | Scan through a string, looking for any | | | location where this RE matches. | +------------------+-----------------------------------------------+ +| ``prefixmatch()``| Determine if the RE matches at the beginning | +| | of the string. Previously named :ref:`match() | +| | `. | ++------------------+-----------------------------------------------+ | ``findall()`` | Find all substrings where the RE matches, and | -| | returns them as a list. | +| | return them as a list. | +------------------+-----------------------------------------------+ | ``finditer()`` | Find all substrings where the RE matches, and | -| | returns them as an :term:`iterator`. | +| | return them as an :term:`iterator`. | +------------------+-----------------------------------------------+ -:meth:`~re.Pattern.match` and :meth:`~re.Pattern.search` return ``None`` if no match can be found. If +:meth:`~re.Pattern.search` and :meth:`~re.Pattern.prefixmatch` return ``None`` if no match can be found. If they're successful, a :ref:`match object ` instance is returned, containing information about the match: where it starts and ends, the substring it matched, and more. @@ -393,19 +394,19 @@ Python interpreter, import the :mod:`re` module, and compile a RE:: Now, you can try matching various strings against the RE ``[a-z]+``. An empty string shouldn't match at all, since ``+`` means 'one or more repetitions'. -:meth:`~re.Pattern.match` should return ``None`` in this case, which will cause the +:meth:`~re.Pattern.search` should return ``None`` in this case, which will cause the interpreter to print no output. You can explicitly print the result of -:meth:`!match` to make this clear. :: +:meth:`!search` to make this clear. :: - >>> p.match("") - >>> print(p.match("")) + >>> p.search("") + >>> print(p.search("")) None Now, let's try it on a string that it should match, such as ``tempo``. In this -case, :meth:`~re.Pattern.match` will return a :ref:`match object `, so you +case, :meth:`~re.Pattern.search` will return a :ref:`match object `, so you should store the result in a variable for later use. :: - >>> m = p.match('tempo') + >>> m = p.search('tempo') >>> m @@ -437,27 +438,28 @@ Trying these methods will soon clarify their meaning:: :meth:`~re.Match.group` returns the substring that was matched by the RE. :meth:`~re.Match.start` and :meth:`~re.Match.end` return the starting and ending index of the match. :meth:`~re.Match.span` -returns both start and end indexes in a single tuple. Since the :meth:`~re.Pattern.match` -method only checks if the RE matches at the start of a string, :meth:`!start` -will always be zero. However, the :meth:`~re.Pattern.search` method of patterns -scans through the string, so the match may not start at zero in that -case. :: +returns both start and end indexes in a single tuple. +The :meth:`~re.Pattern.search` method of patterns +scans through the string, so the match may not start at zero. +However, the :meth:`~re.Pattern.prefixmatch` +method only checks if the RE matches at the start of a string, so :meth:`!start` +will always be zero in that case. :: - >>> print(p.match('::: message')) - None >>> m = p.search('::: message'); print(m) >>> m.group() 'message' >>> m.span() (4, 11) + >>> print(p.prefixmatch('::: message')) + None In actual programs, the most common style is to store the :ref:`match object ` in a variable, and then check if it was ``None``. This usually looks like:: p = re.compile( ... ) - m = p.match( 'string goes here' ) + m = p.search( 'string goes here' ) if m: print('Match found: ', m.group()) else: @@ -473,7 +475,7 @@ Two pattern methods return all of the matches for a pattern. The ``r`` prefix, making the literal a raw string literal, is needed in this example because escape sequences in a normal "cooked" string literal that are not recognized by Python, as opposed to regular expressions, now result in a -:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`. See +:exc:`SyntaxWarning` and will eventually become a :exc:`SyntaxError`. See :ref:`the-backslash-plague`. :meth:`~re.Pattern.findall` has to create the entire list before it can be returned as the @@ -491,19 +493,19 @@ result. The :meth:`~re.Pattern.finditer` method returns a sequence of (29, 31) -Module-Level Functions +Module-level functions ---------------------- You don't have to create a pattern object and call its methods; the -:mod:`re` module also provides top-level functions called :func:`~re.match`, -:func:`~re.search`, :func:`~re.findall`, :func:`~re.sub`, and so forth. These functions +:mod:`re` module also provides top-level functions called :func:`~re.search`, +:func:`~re.prefixmatch`, :func:`~re.findall`, :func:`~re.sub`, and so forth. These functions take the same arguments as the corresponding pattern method with the RE string added as the first argument, and still return either ``None`` or a :ref:`match object ` instance. :: - >>> print(re.match(r'From\s+', 'Fromage amk')) + >>> print(re.prefixmatch(r'From\s+', 'Fromage amk')) None - >>> re.match(r'From\s+', 'From amk Thu May 14 19:12:10 1998') #doctest: +ELLIPSIS + >>> re.prefixmatch(r'From\s+', 'From amk Thu May 14 19:12:10 1998') #doctest: +ELLIPSIS Under the hood, these functions simply create a pattern object for you @@ -518,7 +520,7 @@ Outside of loops, there's not much difference thanks to the internal cache. -Compilation Flags +Compilation flags ----------------- .. currentmodule:: re @@ -642,7 +644,7 @@ of each one. whitespace is in a character class or preceded by an unescaped backslash; this lets you organize and indent the RE more clearly. This flag also lets you put comments within a RE that will be ignored by the engine; comments are marked by - a ``'#'`` that's neither in a character class or preceded by an unescaped + a ``'#'`` that's neither in a character class nor preceded by an unescaped backslash. For example, here's a RE that uses :const:`re.VERBOSE`; see how much easier it @@ -669,7 +671,7 @@ of each one. to understand than the version using :const:`re.VERBOSE`. -More Pattern Power +More pattern power ================== So far we've only covered a part of the features of regular expressions. In @@ -679,7 +681,7 @@ retrieve portions of the text that was matched. .. _more-metacharacters: -More Metacharacters +More metacharacters ------------------- There are some metacharacters that we haven't covered yet. Most of them will be @@ -812,7 +814,7 @@ of a group with a quantifier, such as ``*``, ``+``, ``?``, or ``ab``. :: >>> p = re.compile('(ab)*') - >>> print(p.match('ababababab').span()) + >>> print(p.search('ababababab').span()) (0, 10) Groups indicated with ``'('``, ``')'`` also capture the starting and ending @@ -825,7 +827,7 @@ argument. Later we'll see how to express groups that don't capture the span of text that they match. :: >>> p = re.compile('(a)b') - >>> m = p.match('ab') + >>> m = p.search('ab') >>> m.group() 'ab' >>> m.group(0) @@ -836,7 +838,7 @@ to determine the number, just count the opening parenthesis characters, going from left to right. :: >>> p = re.compile('(a(b)c)d') - >>> m = p.match('abcd') + >>> m = p.search('abcd') >>> m.group(0) 'abcd' >>> m.group(1) @@ -875,7 +877,7 @@ Backreferences like this aren't often useful for just searching through a string find out that they're *very* useful when performing string substitutions. -Non-capturing and Named Groups +Non-capturing and named groups ------------------------------ Elaborate REs may use many groups, both to capture substrings of interest, and @@ -912,10 +914,10 @@ but aren't interested in retrieving the group's contents. You can make this fact explicit by using a non-capturing group: ``(?:...)``, where you can replace the ``...`` with any other regular expression. :: - >>> m = re.match("([abc])+", "abc") + >>> m = re.search("([abc])+", "abc") >>> m.groups() ('c',) - >>> m = re.match("(?:[abc])+", "abc") + >>> m = re.search("(?:[abc])+", "abc") >>> m.groups() () @@ -949,7 +951,7 @@ given numbers, so you can retrieve information about a group in two ways:: Additionally, you can retrieve named groups as a dictionary with :meth:`~re.Match.groupdict`:: - >>> m = re.match(r'(?P\w+) (?P\w+)', 'Jane Doe') + >>> m = re.search(r'(?P\w+) (?P\w+)', 'Jane Doe') >>> m.groupdict() {'first': 'Jane', 'last': 'Doe'} @@ -979,7 +981,7 @@ current point. The regular expression for finding doubled words, 'the the' -Lookahead Assertions +Lookahead assertions -------------------- Another zero-width assertion is the lookahead assertion. Lookahead assertions @@ -1061,7 +1063,7 @@ end in either ``bat`` or ``exe``: ``.*[.](?!bat$|exe$)[^.]*$`` -Modifying Strings +Modifying strings ================= Up to this point, we've simply performed searches against a static string. @@ -1083,7 +1085,7 @@ using the following pattern methods: +------------------+-----------------------------------------------+ -Splitting Strings +Splitting strings ----------------- The :meth:`~re.Pattern.split` method of a pattern splits a string apart @@ -1137,7 +1139,7 @@ argument, but is otherwise the same. :: ['Words', 'words, words.'] -Search and Replace +Search and replace ------------------ Another common task is to find all the matches for a pattern, and replace them @@ -1236,7 +1238,7 @@ pattern object as the first parameter, or use embedded modifiers in the pattern string, e.g. ``sub("(?i)b+", "x", "bbbb BBBB")`` returns ``'x x'``. -Common Problems +Common problems =============== Regular expressions are a powerful tool for some applications, but in some ways @@ -1244,7 +1246,7 @@ their behaviour isn't intuitive and at times they don't behave the way you may expect them to. This section will point out some of the most common pitfalls. -Use String Methods +Use string methods ------------------ Sometimes using the :mod:`re` module is a mistake. If you're matching a fixed @@ -1274,21 +1276,26 @@ In short, before turning to the :mod:`re` module, consider whether your problem can be solved with a faster and simpler string method. -match() versus search() ------------------------ +.. _match-versus-search: -The :func:`~re.match` function only checks if the RE matches at the beginning of the -string while :func:`~re.search` will scan forward through the string for a match. -It's important to keep this distinction in mind. Remember, :func:`!match` will -only report a successful match which will start at 0; if the match wouldn't -start at zero, :func:`!match` will *not* report it. :: +prefixmatch() (aka match) versus search() +----------------------------------------- - >>> print(re.match('super', 'superstition').span()) +:func:`~re.prefixmatch` was added in Python 3.15 as the :ref:`preferred name +` for :func:`~re.match`. Before this, it was only known +as :func:`!match` and the distinction with :func:`~re.search` was often +misunderstood. + +:func:`!prefixmatch` aka :func:`!match` only checks if the RE matches at the +beginning of the string while :func:`!search` scans forward through the +string for a match. :: + + >>> print(re.prefixmatch('super', 'superstition').span()) (0, 5) - >>> print(re.match('super', 'insuperable')) + >>> print(re.prefixmatch('super', 'insuperable')) None -On the other hand, :func:`~re.search` will scan forward through the string, +On the other hand, :func:`~re.search` scans forward through the string, reporting the first match it finds. :: >>> print(re.search('super', 'superstition').span()) @@ -1296,21 +1303,11 @@ reporting the first match it finds. :: >>> print(re.search('super', 'insuperable').span()) (2, 7) -Sometimes you'll be tempted to keep using :func:`re.match`, and just add ``.*`` -to the front of your RE. Resist this temptation and use :func:`re.search` -instead. The regular expression compiler does some analysis of REs in order to -speed up the process of looking for a match. One such analysis figures out what -the first character of a match must be; for example, a pattern starting with -``Crow`` must match starting with a ``'C'``. The analysis lets the engine -quickly scan through the string looking for the starting character, only trying -the full match if a ``'C'`` is found. - -Adding ``.*`` defeats this optimization, requiring scanning to the end of the -string and then backtracking to find a match for the rest of the RE. Use -:func:`re.search` instead. +This distinction is important to remember when using the old :func:`~re.match` +name in code requiring compatibility with older Python versions. -Greedy versus Non-Greedy +Greedy versus non-greedy ------------------------ When repeating a regular expression, as in ``a*``, the resulting action is to @@ -1322,9 +1319,9 @@ doesn't work because of the greedy nature of ``.*``. :: >>> s = 'Title' >>> len(s) 32 - >>> print(re.match('<.*>', s).span()) + >>> print(re.prefixmatch('<.*>', s).span()) (0, 32) - >>> print(re.match('<.*>', s).group()) + >>> print(re.prefixmatch('<.*>', s).group()) Title The RE matches the ``'<'`` in ``''``, and the ``.*`` consumes the rest of @@ -1340,7 +1337,7 @@ example, the ``'>'`` is tried immediately after the first ``'<'`` matches, and when it fails, the engine advances a character at a time, retrying the ``'>'`` at every step. This produces just the right result:: - >>> print(re.match('<.*?>', s).group()) + >>> print(re.prefixmatch('<.*?>', s).group()) (Note that parsing HTML or XML with regular expressions is painful. @@ -1388,9 +1385,9 @@ Feedback ======== Regular expressions are a complicated topic. Did this document help you -understand them? Were there parts that were unclear, or Problems you +understand them? Were there parts that were unclear, or problems you encountered that weren't covered here? If so, please send suggestions for -improvements to the author. +improvements to the :ref:`issue tracker `. The most complete book on regular expressions is almost certainly Jeffrey Friedl's Mastering Regular Expressions, published by O'Reilly. Unfortunately, diff --git a/Doc/howto/remote_debugging.rst b/Doc/howto/remote_debugging.rst index b7323803654..1d5cf24d062 100644 --- a/Doc/howto/remote_debugging.rst +++ b/Doc/howto/remote_debugging.rst @@ -3,6 +3,88 @@ Remote debugging attachment protocol ==================================== +This protocol enables external tools to attach to a running CPython process and +execute Python code remotely. + +Most platforms require elevated privileges to attach to another Python process. + +Disabling remote debugging +-------------------------- + +To disable remote debugging support, use any of the following: + +* Set the :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` environment variable to ``1`` before + starting the interpreter. +* Use the :option:`-X disable_remote_debug` command-line option. +* Compile Python with the :option:`--without-remote-debug` build flag. + +.. _permission-requirements: + +Permission requirements +======================= + +Attaching to a running Python process for remote debugging requires elevated +privileges on most platforms. The specific requirements and troubleshooting +steps depend on your operating system: + +.. rubric:: Linux + +The tracer process must have the ``CAP_SYS_PTRACE`` capability or equivalent +privileges. You can only trace processes you own and can signal. Tracing may +fail if the process is already being traced, or if it is running with +set-user-ID or set-group-ID. Security modules like Yama may further restrict +tracing. + +To temporarily relax ptrace restrictions (until reboot), run: + + ``echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope`` + +.. note:: + + Disabling ``ptrace_scope`` reduces system hardening and should only be done + in trusted environments. + +If running inside a container, use ``--cap-add=SYS_PTRACE`` or +``--privileged``, and run as root if needed. + +Try re-running the command with elevated privileges: + + ``sudo -E !!`` + + +.. rubric:: macOS + +To attach to another process, you typically need to run your debugging tool +with elevated privileges. This can be done by using ``sudo`` or running as +root. + +Even when attaching to processes you own, macOS may block debugging unless +the debugger is run with root privileges due to system security restrictions. + + +.. rubric:: Windows + +To attach to another process, you usually need to run your debugging tool +with administrative privileges. Start the command prompt or terminal as +Administrator. + +Some processes may still be inaccessible even with Administrator rights, +unless you have the ``SeDebugPrivilege`` privilege enabled. + +To resolve file or folder access issues, adjust the security permissions: + + 1. Right-click the file or folder and select **Properties**. + 2. Go to the **Security** tab to view users and groups with access. + 3. Click **Edit** to modify permissions. + 4. Select your user account. + 5. In **Permissions**, check **Read** or **Full control** as needed. + 6. Click **Apply**, then **OK** to confirm. + + +.. note:: + + Ensure you've satisfied all :ref:`permission-requirements` before proceeding. + This section describes the low-level protocol that enables external tools to inject and execute a Python script within a running CPython process. @@ -543,3 +625,57 @@ To inject and execute a Python script in a remote process: 7. Resume the process (if suspended). The script will execute at the next safe evaluation point. +.. _remote-debugging-threat-model: + +Security and threat model +========================= + +The remote debugging protocol relies on the same operating system primitives +used by native debuggers such as GDB and LLDB. Attaching to a process +requires the **same privileges** that those debuggers require, for example +``ptrace`` / Yama LSM on Linux, ``task_for_pid`` on macOS, and +``SeDebugPrivilege`` on Windows. Python does not introduce any new privilege +escalation path; if an attacker already possesses the permissions needed to +attach to a process, they could equally use GDB to read memory or inject +code. + +The following principles define what is, and is not, considered a security +vulnerability in this feature: + +Attaching requires OS-level privileges + On every supported platform the operating system gates cross-process + memory access behind privilege checks (``CAP_SYS_PTRACE``, root, or + administrator rights). A report that demonstrates an issue only after + these privileges have already been obtained is **not** a vulnerability in + CPython, since the OS security boundary was already crossed. + +Crashes or memory errors when reading a compromised process are not vulnerabilities + A tool that reads internal interpreter state from a target process must + trust that memory to be well-formed. If the target process has been + corrupted or is controlled by an attacker, the debugger or profiler may + crash, produce garbage output, or behave unpredictably. This is the same + risk accepted by every ``ptrace``-based debugger. Bugs in this category + (buffer overflows, segmentation faults, or undefined behaviour triggered + by reading corrupted state) are **not** treated as security issues, though + fixes that improve robustness are welcome. + +Vulnerabilities in the target process are not in scope + If the Python process being debugged has already been compromised, the + attacker already controls execution in that process. Demonstrating further + impact from that starting point does not constitute a vulnerability in the + remote debugging protocol. + +When to use ``PYTHON_DISABLE_REMOTE_DEBUG`` +------------------------------------------- + +The environment variable :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` (and the +equivalent :option:`-X disable_remote_debug` flag) allows operators to disable +the in-process side of the protocol as a **defence-in-depth** measure. This +may be useful in hardened or sandboxed deployment environments where no +debugging or profiling of the process is expected and reducing attack surface +is a priority, even though the OS-level privilege checks already prevent +unprivileged access. + +Setting this variable does **not** affect other OS-level debugging interfaces +(``ptrace``, ``/proc``, ``task_for_pid``, etc.), which remain available +according to their own permission models. diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index 254fe729355..243cc27bac7 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -352,6 +352,8 @@ If you don't include such a comment, the default encoding used will be UTF-8 as already mentioned. See also :pep:`263` for more information. +.. _unicode-properties: + Unicode Properties ------------------ diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index d79d1abe8d0..4e77d2cb407 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -15,7 +15,7 @@ Introduction You may also find useful the following article on fetching web resources with Python: - * `Basic Authentication `_ + * `Basic Authentication `__ A tutorial on *Basic Authentication*, with examples in Python. diff --git a/Doc/improve-page-nojs.rst b/Doc/improve-page-nojs.rst new file mode 100644 index 00000000000..91b3a88b95d --- /dev/null +++ b/Doc/improve-page-nojs.rst @@ -0,0 +1,29 @@ +:orphan: + +**************************** +Improve a documentation page +**************************** + +.. This is the no-javascript version of this page. The one most people + will see (with JavaScript enabled) is improve-page.rst. If you edit + this page, please also edit that one, and vice versa. + +.. only:: html and not epub + +We are always interested to hear ideas about improvements to the documentation. + +.. only:: translation + + If the bug or suggested improvement concerns the translation of this + documentation, open an issue or edit the page in + `translation's repository `_ instead. + +You have a few ways to ask questions or suggest changes: + +- You can start a discussion about the page on the Python discussion forum. + This link will start a topic in the Documentation category: + `New Documentation topic `_. + +- You can open an issue on the Python GitHub issue tracker. This link will + create a new issue with the "docs" label: + `New docs issue `_. diff --git a/Doc/improve-page.rst b/Doc/improve-page.rst new file mode 100644 index 00000000000..dc89fcb22fb --- /dev/null +++ b/Doc/improve-page.rst @@ -0,0 +1,65 @@ +:orphan: + +**************************** +Improve a documentation page +**************************** + +.. This is the JavaScript-enabled version of this page. Another version + (for those with JavaScript disabled) is improve-page-nojs.rst. If you + edit this page, please also edit that one, and vice versa. + +.. only:: html and not epub + + .. raw:: html + + + +We are always interested to hear ideas about improvements to the documentation. + +You were reading "PAGETITLE" at ``_. The source for that page is on +`GitHub `_. + +.. only:: translation + + If the bug or suggested improvement concerns the translation of this + documentation, open an issue or edit the page in + `translation's repository `_ instead. + +You have a few ways to ask questions or suggest changes: + +- You can start a discussion about the page on the Python discussion forum. + This link will start a pre-populated topic: + `Question about page "PAGETITLE" `_. + +- You can open an issue on the Python GitHub issue tracker. This link will + create a new pre-populated issue: + `Docs: problem with page "PAGETITLE" `_. + +- You can `edit the page on GitHub `_ + to open a pull request and begin the contribution process. diff --git a/Doc/includes/capi-extension/spammodule-01.c b/Doc/includes/capi-extension/spammodule-01.c new file mode 100644 index 00000000000..0bc34ef5744 --- /dev/null +++ b/Doc/includes/capi-extension/spammodule-01.c @@ -0,0 +1,58 @@ +/* This file needs to be kept in sync with the tutorial + * at Doc/extending/first-extension-module.rst + */ + +/// Includes + +#include +#include // for system() + +/// Implementation of spam.system + +static PyObject * +spam_system(PyObject *self, PyObject *arg) +{ + const char *command = PyUnicode_AsUTF8AndSize(arg, NULL); + if (command == NULL) { + return NULL; + } + int status = system(command); + PyObject *result = PyLong_FromLong(status); + return result; +} + +/// Module method table + +static PyMethodDef spam_methods[] = { + { + .ml_name="system", + .ml_meth=spam_system, + .ml_flags=METH_O, + .ml_doc="Execute a shell command.", + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +/// Module slot table + +PyABIInfo_VAR(abi_info); + +static PyModuleDef_Slot spam_slots[] = { + {Py_mod_abi, &abi_info}, + {Py_mod_name, "spam"}, + {Py_mod_doc, "A wonderful module with an example function"}, + {Py_mod_methods, spam_methods}, + {0, NULL} +}; + +/// Export hook prototype + +PyMODEXPORT_FUNC PyModExport_spam(void); + +/// Module export hook + +PyMODEXPORT_FUNC +PyModExport_spam(void) +{ + return spam_slots; +} diff --git a/Doc/includes/diff.py b/Doc/includes/diff.py index 001619f5f83..bc4bd58ff3e 100644 --- a/Doc/includes/diff.py +++ b/Doc/includes/diff.py @@ -1,4 +1,4 @@ -""" Command line interface to difflib.py providing diffs in four formats: +""" Command-line interface to difflib.py providing diffs in four formats: * ndiff: lists every line and highlights interline changes. * context: highlights clusters of changes in a before/after format. @@ -8,11 +8,11 @@ """ import sys, os, difflib, argparse -from datetime import datetime, timezone +import datetime as dt def file_mtime(path): - t = datetime.fromtimestamp(os.stat(path).st_mtime, - timezone.utc) + t = dt.datetime.fromtimestamp(os.stat(path).st_mtime, + dt.timezone.utc) return t.astimezone().isoformat() def main(): diff --git a/Doc/includes/optional-module.rst b/Doc/includes/optional-module.rst new file mode 100644 index 00000000000..262e73f2eaa --- /dev/null +++ b/Doc/includes/optional-module.rst @@ -0,0 +1,9 @@ +This is an :term:`optional module`. +If it is missing from your copy of CPython, +look for documentation from your distributor (that is, +whoever provided Python to you). +If you are the distributor, see :ref:`optional-module-requirements`. + +.. Similar notes appear in the docs of the modules: + - zipfile + - tarfile diff --git a/Doc/includes/tzinfo_examples.py b/Doc/includes/tzinfo_examples.py index 1fa6e615e46..762b1b62fc8 100644 --- a/Doc/includes/tzinfo_examples.py +++ b/Doc/includes/tzinfo_examples.py @@ -1,68 +1,70 @@ -from datetime import tzinfo, timedelta, datetime - -ZERO = timedelta(0) -HOUR = timedelta(hours=1) -SECOND = timedelta(seconds=1) +import datetime as dt # A class capturing the platform's idea of local time. # (May result in wrong values on historical times in # timezones where UTC offset and/or the DST rules had # changed in the past.) -import time as _time +import time -STDOFFSET = timedelta(seconds = -_time.timezone) -if _time.daylight: - DSTOFFSET = timedelta(seconds = -_time.altzone) +ZERO = dt.timedelta(0) +HOUR = dt.timedelta(hours=1) +SECOND = dt.timedelta(seconds=1) + +STDOFFSET = dt.timedelta(seconds=-time.timezone) +if time.daylight: + DSTOFFSET = dt.timedelta(seconds=-time.altzone) else: DSTOFFSET = STDOFFSET DSTDIFF = DSTOFFSET - STDOFFSET -class LocalTimezone(tzinfo): - def fromutc(self, dt): - assert dt.tzinfo is self - stamp = (dt - datetime(1970, 1, 1, tzinfo=self)) // SECOND - args = _time.localtime(stamp)[:6] +class LocalTimezone(dt.tzinfo): + + def fromutc(self, when): + assert when.tzinfo is self + stamp = (when - dt.datetime(1970, 1, 1, tzinfo=self)) // SECOND + args = time.localtime(stamp)[:6] dst_diff = DSTDIFF // SECOND # Detect fold - fold = (args == _time.localtime(stamp - dst_diff)) - return datetime(*args, microsecond=dt.microsecond, - tzinfo=self, fold=fold) + fold = (args == time.localtime(stamp - dst_diff)) + return dt.datetime(*args, microsecond=when.microsecond, + tzinfo=self, fold=fold) - def utcoffset(self, dt): - if self._isdst(dt): + def utcoffset(self, when): + if self._isdst(when): return DSTOFFSET else: return STDOFFSET - def dst(self, dt): - if self._isdst(dt): + def dst(self, when): + if self._isdst(when): return DSTDIFF else: return ZERO - def tzname(self, dt): - return _time.tzname[self._isdst(dt)] + def tzname(self, when): + return time.tzname[self._isdst(when)] - def _isdst(self, dt): - tt = (dt.year, dt.month, dt.day, - dt.hour, dt.minute, dt.second, - dt.weekday(), 0, 0) - stamp = _time.mktime(tt) - tt = _time.localtime(stamp) + def _isdst(self, when): + tt = (when.year, when.month, when.day, + when.hour, when.minute, when.second, + when.weekday(), 0, 0) + stamp = time.mktime(tt) + tt = time.localtime(stamp) return tt.tm_isdst > 0 + Local = LocalTimezone() # A complete implementation of current DST rules for major US time zones. -def first_sunday_on_or_after(dt): - days_to_go = 6 - dt.weekday() +def first_sunday_on_or_after(when): + days_to_go = 6 - when.weekday() if days_to_go: - dt += timedelta(days_to_go) - return dt + when += dt.timedelta(days_to_go) + return when # US DST Rules @@ -75,21 +77,22 @@ def first_sunday_on_or_after(dt): # # In the US, since 2007, DST starts at 2am (standard time) on the second # Sunday in March, which is the first Sunday on or after Mar 8. -DSTSTART_2007 = datetime(1, 3, 8, 2) +DSTSTART_2007 = dt.datetime(1, 3, 8, 2) # and ends at 2am (DST time) on the first Sunday of Nov. -DSTEND_2007 = datetime(1, 11, 1, 2) +DSTEND_2007 = dt.datetime(1, 11, 1, 2) # From 1987 to 2006, DST used to start at 2am (standard time) on the first # Sunday in April and to end at 2am (DST time) on the last # Sunday of October, which is the first Sunday on or after Oct 25. -DSTSTART_1987_2006 = datetime(1, 4, 1, 2) -DSTEND_1987_2006 = datetime(1, 10, 25, 2) +DSTSTART_1987_2006 = dt.datetime(1, 4, 1, 2) +DSTEND_1987_2006 = dt.datetime(1, 10, 25, 2) # From 1967 to 1986, DST used to start at 2am (standard time) on the last # Sunday in April (the one on or after April 24) and to end at 2am (DST time) # on the last Sunday of October, which is the first Sunday # on or after Oct 25. -DSTSTART_1967_1986 = datetime(1, 4, 24, 2) +DSTSTART_1967_1986 = dt.datetime(1, 4, 24, 2) DSTEND_1967_1986 = DSTEND_1987_2006 + def us_dst_range(year): # Find start and end times for US DST. For years before 1967, return # start = end for no DST. @@ -100,17 +103,17 @@ def us_dst_range(year): elif 1966 < year < 1987: dststart, dstend = DSTSTART_1967_1986, DSTEND_1967_1986 else: - return (datetime(year, 1, 1), ) * 2 + return (dt.datetime(year, 1, 1), ) * 2 start = first_sunday_on_or_after(dststart.replace(year=year)) end = first_sunday_on_or_after(dstend.replace(year=year)) return start, end -class USTimeZone(tzinfo): +class USTimeZone(dt.tzinfo): def __init__(self, hours, reprname, stdname, dstname): - self.stdoffset = timedelta(hours=hours) + self.stdoffset = dt.timedelta(hours=hours) self.reprname = reprname self.stdname = stdname self.dstname = dstname @@ -118,45 +121,45 @@ def __init__(self, hours, reprname, stdname, dstname): def __repr__(self): return self.reprname - def tzname(self, dt): - if self.dst(dt): + def tzname(self, when): + if self.dst(when): return self.dstname else: return self.stdname - def utcoffset(self, dt): - return self.stdoffset + self.dst(dt) + def utcoffset(self, when): + return self.stdoffset + self.dst(when) - def dst(self, dt): - if dt is None or dt.tzinfo is None: + def dst(self, when): + if when is None or when.tzinfo is None: # An exception may be sensible here, in one or both cases. # It depends on how you want to treat them. The default # fromutc() implementation (called by the default astimezone() - # implementation) passes a datetime with dt.tzinfo is self. + # implementation) passes a datetime with when.tzinfo is self. return ZERO - assert dt.tzinfo is self - start, end = us_dst_range(dt.year) + assert when.tzinfo is self + start, end = us_dst_range(when.year) # Can't compare naive to aware objects, so strip the timezone from - # dt first. - dt = dt.replace(tzinfo=None) - if start + HOUR <= dt < end - HOUR: + # when first. + when = when.replace(tzinfo=None) + if start + HOUR <= when < end - HOUR: # DST is in effect. return HOUR - if end - HOUR <= dt < end: - # Fold (an ambiguous hour): use dt.fold to disambiguate. - return ZERO if dt.fold else HOUR - if start <= dt < start + HOUR: + if end - HOUR <= when < end: + # Fold (an ambiguous hour): use when.fold to disambiguate. + return ZERO if when.fold else HOUR + if start <= when < start + HOUR: # Gap (a non-existent hour): reverse the fold rule. - return HOUR if dt.fold else ZERO + return HOUR if when.fold else ZERO # DST is off. return ZERO - def fromutc(self, dt): - assert dt.tzinfo is self - start, end = us_dst_range(dt.year) + def fromutc(self, when): + assert when.tzinfo is self + start, end = us_dst_range(when.year) start = start.replace(tzinfo=self) end = end.replace(tzinfo=self) - std_time = dt + self.stdoffset + std_time = when + self.stdoffset dst_time = std_time + HOUR if end <= dst_time < end + HOUR: # Repeated hour diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index 3a485a43a5a..c372d9f4741 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -1,16 +1,14 @@ -.. highlight:: none +.. highlight:: shell .. _installing-index: ************************* -Installing Python Modules +Installing Python modules ************************* -:Email: distutils-sig@python.org - As a popular open source development project, Python has an active supporting community of contributors and users that also make their software -available for other Python developers to use under open source license terms. +available for other Python developers to use under open-source license terms. This allows Python users to share and collaborate effectively, benefiting from the solutions others have already created to common (and sometimes @@ -34,34 +32,24 @@ creating and sharing your own Python projects, refer to the Key terms ========= -* ``pip`` is the preferred installer program. Starting with Python 3.4, it +* :program:`pip` is the preferred installer program. It is included by default with the Python binary installers. * A *virtual environment* is a semi-isolated Python environment that allows packages to be installed for use by a particular application, rather than being installed system wide. -* ``venv`` is the standard tool for creating virtual environments, and has - been part of Python since Python 3.3. Starting with Python 3.4, it - defaults to installing ``pip`` into all created virtual environments. -* ``virtualenv`` is a third party alternative (and predecessor) to - ``venv``. It allows virtual environments to be used on versions of - Python prior to 3.4, which either don't provide ``venv`` at all, or - aren't able to automatically install ``pip`` into created environments. -* The `Python Package Index `__ is a public +* ``venv`` is the standard tool for creating virtual environments. + It defaults to installing :program:`pip` into all created virtual environments. +* ``virtualenv`` is a third-party alternative (and predecessor) to + ``venv``. +* The `Python Package Index (PyPI) `__ is a public repository of open source licensed packages made available for use by other Python users. -* the `Python Packaging Authority +* The `Python Packaging Authority `__ is the group of developers and documentation authors responsible for the maintenance and evolution of the standard packaging tools and the associated metadata and file format standards. They maintain a variety of tools, documentation, and issue trackers on `GitHub `__. -* ``distutils`` is the original build and distribution system first added to - the Python standard library in 1998. While direct use of ``distutils`` is - being phased out, it still laid the foundation for the current packaging - and distribution infrastructure, and it not only remains part of the - standard library, but its name lives on in other ways (such as the name - of the mailing list used to coordinate Python packaging standards - development). .. versionchanged:: 3.5 The use of ``venv`` is now recommended for creating virtual environments. @@ -79,7 +67,7 @@ The standard packaging tools are all designed to be used from the command line. The following command will install the latest version of a module and its -dependencies from the Python Package Index:: +dependencies from PyPI:: python -m pip install SomePackage @@ -106,7 +94,7 @@ explicitly:: python -m pip install --upgrade SomePackage -More information and resources regarding ``pip`` and its capabilities can be +More information and resources regarding :program:`pip` and its capabilities can be found in the `Python Packaging User Guide `__. Creation of virtual environments is done through the :mod:`venv` module. @@ -124,19 +112,6 @@ How do I ...? These are quick answers or links for some common tasks. -... install ``pip`` in versions of Python prior to Python 3.4? --------------------------------------------------------------- - -Python only started bundling ``pip`` with Python 3.4. For earlier versions, -``pip`` needs to be "bootstrapped" as described in the Python Packaging -User Guide. - -.. seealso:: - - `Python Packaging User Guide: Requirements for Installing Packages - `__ - - .. installing-per-user-installation: ... install packages just for the current user? @@ -150,10 +125,10 @@ package just for the current user, rather than for all users of the system. --------------------------------------- A number of scientific Python packages have complex binary dependencies, and -aren't currently easy to install using ``pip`` directly. At this point in -time, it will often be easier for users to install these packages by +aren't currently easy to install using :program:`pip` directly. +It will often be easier for users to install these packages by `other means `__ -rather than attempting to install them with ``pip``. +rather than attempting to install them with :program:`pip`. .. seealso:: @@ -166,22 +141,18 @@ rather than attempting to install them with ``pip``. On Linux, macOS, and other POSIX systems, use the versioned Python commands in combination with the ``-m`` switch to run the appropriate copy of -``pip``:: +:program:`pip`:: - python2 -m pip install SomePackage # default Python 2 - python2.7 -m pip install SomePackage # specifically Python 2.7 - python3 -m pip install SomePackage # default Python 3 - python3.4 -m pip install SomePackage # specifically Python 3.4 + python3 -m pip install SomePackage # default Python 3 + python3.14 -m pip install SomePackage # specifically Python 3.14 -Appropriately versioned ``pip`` commands may also be available. +Appropriately versioned :program:`pip` commands may also be available. -On Windows, use the ``py`` Python launcher in combination with the ``-m`` +On Windows, use the :program:`py` Python launcher in combination with the ``-m`` switch:: - py -2 -m pip install SomePackage # default Python 2 - py -2.7 -m pip install SomePackage # specifically Python 2.7 - py -3 -m pip install SomePackage # default Python 3 - py -3.4 -m pip install SomePackage # specifically Python 3.4 + py -3 -m pip install SomePackage # default Python 3 + py -3.14 -m pip install SomePackage # specifically Python 3.14 .. other questions: @@ -201,39 +172,38 @@ On Linux systems, a Python installation will typically be included as part of the distribution. Installing into this Python installation requires root access to the system, and may interfere with the operation of the system package manager and other components of the system if a component -is unexpectedly upgraded using ``pip``. +is unexpectedly upgraded using :program:`pip`. On such systems, it is often better to use a virtual environment or a -per-user installation when installing packages with ``pip``. +per-user installation when installing packages with :program:`pip`. Pip not installed ----------------- -It is possible that ``pip`` does not get installed by default. One potential fix is:: +It is possible that :program:`pip` does not get installed by default. One potential fix is:: python -m ensurepip --default-pip -There are also additional resources for `installing pip. -`__ +There are also additional resources for `installing pip +`__. Installing binary extensions ---------------------------- -Python has typically relied heavily on source based distribution, with end +Python once relied heavily on source-based distribution, with end users being expected to compile extension modules from source as part of the installation process. -With the introduction of support for the binary ``wheel`` format, and the -ability to publish wheels for at least Windows and macOS through the -Python Package Index, this problem is expected to diminish over time, +With the introduction of the binary wheel format, and the +ability to publish wheels through PyPI, this problem is diminishing, as users are more regularly able to install pre-built extensions rather than needing to build them themselves. Some of the solutions for installing `scientific software `__ -that are not yet available as pre-built ``wheel`` files may also help with +that are not yet available as pre-built wheel files may also help with obtaining other binary extensions without needing to build them locally. .. seealso:: diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index 5d916b30112..749e4543c5b 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -15,7 +15,7 @@ before the release in which the feature becomes standard. While these future statements are given additional special meaning by the Python compiler, they are still executed like any other import statement and -the :mod:`__future__` exists and is handled by the import system the same way +the :mod:`!__future__` exists and is handled by the import system the same way any other Python module would be. This design serves three purposes: * To avoid confusing existing tools that analyze import statements and expect to @@ -23,17 +23,17 @@ any other Python module would be. This design serves three purposes: * To document when incompatible changes were introduced, and when they will be --- or were --- made mandatory. This is a form of executable documentation, and - can be inspected programmatically via importing :mod:`__future__` and examining + can be inspected programmatically via importing :mod:`!__future__` and examining its contents. * To ensure that :ref:`future statements ` run under releases prior to - Python 2.1 at least yield runtime exceptions (the import of :mod:`__future__` + Python 2.1 at least yield runtime exceptions (the import of :mod:`!__future__` will fail, because there was no module of that name prior to 2.1). Module Contents --------------- -No feature description will ever be deleted from :mod:`__future__`. Since its +No feature description will ever be deleted from :mod:`!__future__`. Since its introduction in Python 2.1 the following features have found their way into the language using this mechanism: diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index 1d00d05817e..47f5eabb6f2 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -127,7 +127,7 @@ This module defines the following constants and functions: .. versionchanged:: 3.13 Added support for GNU/kFreeBSD. - .. versionchanged:: next + .. versionchanged:: 3.15 Added support for Solaris. diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 49e541a9d9b..8112cfee7d2 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -4,10 +4,6 @@ .. module:: abc :synopsis: Abstract base classes according to :pep:`3119`. -.. moduleauthor:: Guido van Rossum -.. sectionauthor:: Georg Brandl -.. much of the content adapted from docstrings - **Source code:** :source:`Lib/abc.py` -------------- diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index d6f5055955e..af28fe0e2fd 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -340,14 +340,29 @@ Functions * VALUE: :attr:`!object.__annotations__` is tried first; if that does not exist, the :attr:`!object.__annotate__` function is called if it exists. + * FORWARDREF: If :attr:`!object.__annotations__` exists and can be evaluated successfully, it is used; otherwise, the :attr:`!object.__annotate__` function is called. If it does not exist either, :attr:`!object.__annotations__` is tried again and any error from accessing it is re-raised. + + * When calling :attr:`!object.__annotate__` it is first called with :attr:`~Format.FORWARDREF`. + If this is not implemented, it will then check if :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` + is supported and use that in the fake globals environment. + If neither of these formats are supported, it will fall back to using :attr:`~Format.VALUE`. + If :attr:`~Format.VALUE` fails, the error from this call will be raised. + * STRING: If :attr:`!object.__annotate__` exists, it is called first; otherwise, :attr:`!object.__annotations__` is used and stringified using :func:`annotations_to_string`. + * When calling :attr:`!object.__annotate__` it is first called with :attr:`~Format.STRING`. + If this is not implemented, it will then check if :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` + is supported and use that in the fake globals environment. + If neither of these formats are supported, it will fall back to using :attr:`~Format.VALUE` + with the result converted using :func:`annotations_to_string`. + If :attr:`~Format.VALUE` fails, the error from this call will be raised. + Returns a dict. :func:`!get_annotations` returns a new dict every time it's called; calling it twice on the same object will return two different but equivalent dicts. @@ -495,6 +510,81 @@ annotations from the class and puts them in a separate attribute: return typ +Creating a custom callable annotate function +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Custom :term:`annotate functions ` may be literal functions like those +automatically generated for functions, classes, and modules. Or, they may wish to utilise +the encapsulation provided by classes, in which case any :term:`callable` can be used as +an :term:`annotate function`. + +To provide the :attr:`~Format.VALUE`, :attr:`~Format.STRING`, or +:attr:`~Format.FORWARDREF` formats directly, an :term:`annotate function` must provide +the following attribute: + +* A callable ``__call__`` with signature ``__call__(format, /) -> dict``, that does not + raise a :exc:`NotImplementedError` when called with a supported format. + +To provide the :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` format, which is used to +automatically generate :attr:`~Format.STRING` or :attr:`~Format.FORWARDREF` if they are +not supported directly, :term:`annotate functions ` must provide the +following attributes: + +* A callable ``__call__`` with signature ``__call__(format, /) -> dict``, that does not + raise a :exc:`NotImplementedError` when called with + :attr:`~Format.VALUE_WITH_FAKE_GLOBALS`. +* A :ref:`code object ` ``__code__`` containing the compiled code for the + annotate function. +* Optional: A tuple of the function's positional defaults ``__kwdefaults__``, if the + function represented by ``__code__`` uses any positional defaults. +* Optional: A dict of the function's keyword defaults ``__defaults__``, if the function + represented by ``__code__`` uses any keyword defaults. +* Optional: All other :ref:`function attributes `. + +.. code-block:: python + + class Annotate: + called_formats = [] + + def __call__(self, format=None, /, *, _self=None): + # When called with fake globals, `_self` will be the + # actual self value, and `self` will be the format. + if _self is not None: + self, format = _self, self + + self.called_formats.append(format) + if format <= 2: # VALUE or VALUE_WITH_FAKE_GLOBALS + return {"x": MyType} + raise NotImplementedError + + __code__ = __call__.__code__ + __defaults__ = (None,) + __kwdefaults__ = property(lambda self: dict(_self=self)) + + __globals__ = {} + __builtins__ = {} + __closure__ = None + +This can then be called with: + +.. code-block:: pycon + + >>> from annotationlib import call_annotate_function, Format + >>> call_annotate_function(Annotate(), format=Format.STRING) + {'x': 'MyType'} + +Or used as the annotate function for an object: + +.. code-block:: pycon + + >>> from annotationlib import get_annotations, Format + >>> class C: + ... pass + >>> C.__annotate__ = Annotate() + >>> get_annotations(Annotate(), format=Format.STRING) + {'x': 'MyType'} + + Limitations of the ``STRING`` format ------------------------------------ diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 79e15994491..e37afd6d0b6 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -4,16 +4,13 @@ .. module:: argparse :synopsis: Command-line option and argument parsing library. -.. moduleauthor:: Steven Bethard -.. sectionauthor:: Steven Bethard - .. versionadded:: 3.2 **Source code:** :source:`Lib/argparse.py` .. note:: - While :mod:`argparse` is the default recommended standard library module + While :mod:`!argparse` is the default recommended standard library module for implementing basic command line applications, authors with more exacting requirements for exactly how their command line applications behave may find it doesn't provide the necessary level of control. @@ -74,7 +71,7 @@ ArgumentParser objects prefix_chars='-', fromfile_prefix_chars=None, \ argument_default=None, conflict_handler='error', \ add_help=True, allow_abbrev=True, exit_on_error=True, \ - *, suggest_on_error=False, color=True) + *, suggest_on_error=True, color=True) Create a new :class:`ArgumentParser` object. All parameters should be passed as keyword arguments. Each parameter has its own more detailed description @@ -117,7 +114,7 @@ ArgumentParser objects error info when an error occurs. (default: ``True``) * suggest_on_error_ - Enables suggestions for mistyped argument choices - and subparser names (default: ``False``) + and subparser names (default: ``True``) * color_ - Allow color output (default: ``True``) @@ -134,6 +131,9 @@ ArgumentParser objects .. versionchanged:: 3.14 *suggest_on_error* and *color* parameters were added. + .. versionchanged:: 3.15 + *suggest_on_error* default changed to ``True``. + The following sections describe how each of these are used. @@ -596,32 +596,23 @@ suggest_on_error ^^^^^^^^^^^^^^^^ By default, when a user passes an invalid argument choice or subparser name, -:class:`ArgumentParser` will exit with error info and list the permissible -argument choices (if specified) or subparser names as part of the error message. +:class:`ArgumentParser` will exit with error info and provide suggestions for +mistyped arguments. The error message will list the permissible argument +choices (if specified) or subparser names, along with a "maybe you meant" +suggestion if a close match is found. Note that this only applies for arguments +when the choices specified are strings:: -If the user would like to enable suggestions for mistyped argument choices and -subparser names, the feature can be enabled by setting ``suggest_on_error`` to -``True``. Note that this only applies for arguments when the choices specified -are strings:: + >>> parser = argparse.ArgumentParser(suggest_on_error=True) + >>> parser.add_argument('--action', choices=['debug', 'dryrun']) + >>> parser.parse_args(['--action', 'debugg']) + usage: tester.py [-h] [--action {debug,dryrun}] + tester.py: error: argument --action: invalid choice: 'debugg', maybe you meant 'debug'? (choose from debug, dryrun) - >>> parser = argparse.ArgumentParser(description='Process some integers.', - suggest_on_error=True) - >>> parser.add_argument('--action', choices=['sum', 'max']) - >>> parser.add_argument('integers', metavar='N', type=int, nargs='+', - ... help='an integer for the accumulator') - >>> parser.parse_args(['--action', 'sumn', 1, 2, 3]) - tester.py: error: argument --action: invalid choice: 'sumn', maybe you meant 'sum'? (choose from 'sum', 'max') - -If you're writing code that needs to be compatible with older Python versions -and want to opportunistically use ``suggest_on_error`` when it's available, you -can set it as an attribute after initializing the parser instead of using the -keyword argument:: - - >>> parser = argparse.ArgumentParser(description='Process some integers.') - >>> parser.suggest_on_error = True +You can disable suggestions by setting ``suggest_on_error`` to ``False``. .. versionadded:: 3.14 - +.. versionchanged:: 3.15 + Changed default value of ``suggest_on_error`` from ``False`` to ``True``. color ^^^^^ @@ -639,8 +630,34 @@ by setting ``color`` to ``False``:: ... help='an integer for the accumulator') >>> parser.parse_args(['--help']) +Note that when ``color=True``, colored output depends on both environment +variables and terminal capabilities. However, if ``color=False``, colored +output is always disabled, even if environment variables like ``FORCE_COLOR`` +are set. + .. versionadded:: 3.14 +To highlight inline code in your description or epilog text, you can use +backticks:: + + >>> parser = argparse.ArgumentParser( + ... formatter_class=argparse.RawDescriptionHelpFormatter, + ... epilog='''Examples: + ... `python -m myapp --verbose` + ... `python -m myapp --config settings.json` + ... ''') + +When colors are enabled, the text inside backticks will be displayed in a +distinct color to help examples stand out. When colors are disabled, backticks +are preserved as-is, which is readable in plain text. + +.. note:: + + Backtick markup only applies to description and epilog text. It does not + apply to individual argument ``help`` strings. + +.. versionadded:: 3.15 + The add_argument() method ------------------------- @@ -681,6 +698,8 @@ The add_argument() method * deprecated_ - Whether or not use of the argument is deprecated. + The method returns an :class:`Action` object representing the argument. + The following sections describe how each of these are used. @@ -722,13 +741,13 @@ By default, :mod:`!argparse` automatically handles the internal naming and display names of arguments, simplifying the process without requiring additional configuration. As such, you do not need to specify the dest_ and metavar_ parameters. -The dest_ parameter defaults to the argument name with underscores ``_`` -replacing hyphens ``-`` . The metavar_ parameter defaults to the -upper-cased name. For example:: +For optional arguments, the dest_ parameter defaults to the argument name, with +underscores ``_`` replacing hyphens ``-``. The metavar_ parameter defaults to +the upper-cased name. For example:: >>> parser = argparse.ArgumentParser(prog='PROG') >>> parser.add_argument('--foo-bar') - >>> parser.parse_args(['--foo-bar', 'FOO-BAR'] + >>> parser.parse_args(['--foo-bar', 'FOO-BAR']) Namespace(foo_bar='FOO-BAR') >>> parser.print_help() usage: [-h] [--foo-bar FOO-BAR] @@ -763,9 +782,9 @@ how the command-line arguments should be handled. The supplied actions are: Namespace(foo=42) * ``'store_true'`` and ``'store_false'`` - These are special cases of - ``'store_const'`` used for storing the values ``True`` and ``False`` - respectively. In addition, they create default values of ``False`` and - ``True`` respectively:: + ``'store_const'`` that respectively store the values ``True`` and ``False`` + with default values of ``False`` and + ``True``:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', action='store_true') @@ -774,19 +793,19 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args('--foo --bar'.split()) Namespace(foo=True, bar=False, baz=True) -* ``'append'`` - This stores a list, and appends each argument value to the - list. It is useful to allow an option to be specified multiple times. - If the default value is non-empty, the default elements will be present - in the parsed value for the option, with any values from the - command line appended after those default values. Example usage:: +* ``'append'`` - This appends each argument value to a list. + It is useful for allowing an option to be specified multiple times. + If the default value is a non-empty list, the parsed value will start + with the default list's elements and any values from the command line + will be appended after those default values. Example usage:: >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action='append') + >>> parser.add_argument('--foo', action='append', default=['0']) >>> parser.parse_args('--foo 1 --foo 2'.split()) - Namespace(foo=['1', '2']) + Namespace(foo=['0', '1', '2']) -* ``'append_const'`` - This stores a list, and appends the value specified by - the const_ keyword argument to the list; note that the const_ keyword +* ``'append_const'`` - This appends the value specified by + the const_ keyword argument to a list; note that the const_ keyword argument defaults to ``None``. The ``'append_const'`` action is typically useful when multiple arguments need to store constants to the same list. For example:: @@ -797,8 +816,8 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args('--str --int'.split()) Namespace(types=[, ]) -* ``'extend'`` - This stores a list and appends each item from the multi-value - argument list to it. +* ``'extend'`` - This appends each item from a multi-value + argument to a list. The ``'extend'`` action is typically used with the nargs_ keyword argument value ``'+'`` or ``'*'``. Note that when nargs_ is ``None`` (the default) or ``'?'``, each @@ -812,7 +831,7 @@ how the command-line arguments should be handled. The supplied actions are: .. versionadded:: 3.8 -* ``'count'`` - This counts the number of times a keyword argument occurs. For +* ``'count'`` - This counts the number of times an argument occurs. For example, this is useful for increasing verbosity levels:: >>> parser = argparse.ArgumentParser() @@ -981,8 +1000,8 @@ the various :class:`ArgumentParser` actions. The two most common uses of it are (like ``-f`` or ``--foo``) and ``nargs='?'``. This creates an optional argument that can be followed by zero or one command-line arguments. When parsing the command line, if the option string is encountered with no - command-line argument following it, the value of ``const`` will be assumed to - be ``None`` instead. See the nargs_ description for examples. + command-line argument following it, the value from ``const`` will be used. + See the nargs_ description for examples. .. versionchanged:: 3.11 ``const=None`` by default, including when ``action='append_const'`` or @@ -1099,7 +1118,15 @@ User defined functions can be used as well: The :func:`bool` function is not recommended as a type converter. All it does is convert empty strings to ``False`` and non-empty strings to ``True``. -This is usually not what is desired. +This is usually not what is desired:: + + >>> parser = argparse.ArgumentParser() + >>> _ = parser.add_argument('--verbose', type=bool) + >>> parser.parse_args(['--verbose', 'False']) + Namespace(verbose=True) + +See :class:`BooleanOptionalAction` or ``action='store_true'`` for common +alternatives. In general, the ``type`` keyword is a convenience that should only be used for simple conversions that can only raise one of the three supported exceptions. @@ -1142,16 +1169,21 @@ if the argument was not one of the acceptable values:: game.py: error: argument move: invalid choice: 'fire' (choose from 'rock', 'paper', 'scissors') -Note that inclusion in the *choices* sequence is checked after any type_ -conversions have been performed, so the type of the objects in the *choices* -sequence should match the type_ specified. - Any sequence can be passed as the *choices* value, so :class:`list` objects, :class:`tuple` objects, and custom sequences are all supported. Use of :class:`enum.Enum` is not recommended because it is difficult to control its appearance in usage, help, and error messages. +Note that *choices* are checked after any type_ +conversions have been performed, so objects in *choices* +should match the type_ specified. This can make *choices* +appear unfamiliar in usage, help, or error messages. + +To keep *choices* user-friendly, consider a custom type wrapper that +converts and formats values, or omit type_ and handle conversion in +your application code. + Formatted choices override the default *metavar* which is normally derived from *dest*. This is usually what you want because the user never sees the *dest* parameter. If this display isn't desirable (perhaps because there are @@ -1313,8 +1345,12 @@ attribute is determined by the ``dest`` keyword argument of For optional argument actions, the value of ``dest`` is normally inferred from the option strings. :class:`ArgumentParser` generates the value of ``dest`` by -taking the first long option string and stripping away the initial ``--`` -string. If no long option strings were supplied, ``dest`` will be derived from +taking the first double-dash long option string and stripping away the initial +``-`` characters. +If no double-dash long option strings were supplied, ``dest`` will be derived +from the first single-dash long option string by stripping the initial ``-`` +character. +If no long option strings were supplied, ``dest`` will be derived from the first short option string by stripping the initial ``-`` character. Any internal ``-`` characters will be converted to ``_`` characters to make sure the string is a valid attribute name. The examples below illustrate this @@ -1322,11 +1358,12 @@ behavior:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('-f', '--foo-bar', '--foo') + >>> parser.add_argument('-q', '-quz') >>> parser.add_argument('-x', '-y') - >>> parser.parse_args('-f 1 -x 2'.split()) - Namespace(foo_bar='1', x='2') - >>> parser.parse_args('--foo 1 -y 2'.split()) - Namespace(foo_bar='1', x='2') + >>> parser.parse_args('-f 1 -q 2 -x 3'.split()) + Namespace(foo_bar='1', quz='2', x='3') + >>> parser.parse_args('--foo 1 -quz 2 -y 3'.split()) + Namespace(foo_bar='1', quz='2', x='2') ``dest`` allows a custom attribute name to be provided:: @@ -1335,6 +1372,9 @@ behavior:: >>> parser.parse_args('--foo XXX'.split()) Namespace(bar='XXX') +.. versionchanged:: 3.15 + Single-dash long option now takes precedence over short options. + .. _deprecated: @@ -1428,8 +1468,18 @@ this API may be passed as the ``action`` parameter to >>> parser.parse_args(['--no-foo']) Namespace(foo=False) + Single-dash long options are also supported. + For example, negative option ``-nofoo`` is automatically added for + positive option ``-foo``. + But no additional options are added for short options such as ``-f``. + .. versionadded:: 3.9 + .. versionchanged:: 3.15 + Added support for single-dash options. + + Added support for alternate prefix_chars_. + The parse_args() method ----------------------- @@ -1652,7 +1702,7 @@ The Namespace object Other utilities --------------- -Sub-commands +Subcommands ^^^^^^^^^^^^ .. method:: ArgumentParser.add_subparsers(*, [title], [description], [prog], \ @@ -1681,7 +1731,7 @@ Sub-commands * *description* - description for the sub-parser group in help output, by default ``None`` - * *prog* - usage information that will be displayed with sub-command help, + * *prog* - usage information that will be displayed with subcommand help, by default the name of the program and any positional arguments before the subparser argument @@ -1691,7 +1741,7 @@ Sub-commands * action_ - the basic type of action to be taken when this argument is encountered at the command line - * dest_ - name of the attribute under which sub-command name will be + * dest_ - name of the attribute under which subcommand name will be stored; by default ``None`` and no value is stored * required_ - Whether or not a subcommand must be provided, by default @@ -1723,7 +1773,7 @@ Sub-commands >>> parser.parse_args(['--foo', 'b', '--baz', 'Z']) Namespace(baz='Z', foo=True) - Note that the object returned by :meth:`parse_args` will only contain + Note that the object returned by :meth:`~ArgumentParser.parse_args` will only contain attributes for the main parser and the subparser that was selected by the command line (and not any other subparsers). So in the example above, when the ``a`` command is specified, only the ``foo`` and ``bar`` attributes are @@ -1766,7 +1816,7 @@ Sub-commands -h, --help show this help message and exit --baz {X,Y,Z} baz help - The :meth:`add_subparsers` method also supports ``title`` and ``description`` + The :meth:`~ArgumentParser.add_subparsers` method also supports ``title`` and ``description`` keyword arguments. When either is present, the subparser's commands will appear in their own group in the help output. For example:: @@ -1787,34 +1837,8 @@ Sub-commands {foo,bar} additional help - Furthermore, :meth:`~_SubParsersAction.add_parser` supports an additional - *aliases* argument, - which allows multiple strings to refer to the same subparser. This example, - like ``svn``, aliases ``co`` as a shorthand for ``checkout``:: - - >>> parser = argparse.ArgumentParser() - >>> subparsers = parser.add_subparsers() - >>> checkout = subparsers.add_parser('checkout', aliases=['co']) - >>> checkout.add_argument('foo') - >>> parser.parse_args(['co', 'bar']) - Namespace(foo='bar') - - :meth:`~_SubParsersAction.add_parser` supports also an additional - *deprecated* argument, which allows to deprecate the subparser. - - >>> import argparse - >>> parser = argparse.ArgumentParser(prog='chicken.py') - >>> subparsers = parser.add_subparsers() - >>> run = subparsers.add_parser('run') - >>> fly = subparsers.add_parser('fly', deprecated=True) - >>> parser.parse_args(['fly']) # doctest: +SKIP - chicken.py: warning: command 'fly' is deprecated - Namespace() - - .. versionadded:: 3.13 - One particularly effective way of handling subcommands is to combine the use - of the :meth:`add_subparsers` method with calls to :meth:`set_defaults` so + of the :meth:`~ArgumentParser.add_subparsers` method with calls to :meth:`~ArgumentParser.set_defaults` so that each subparser knows which Python function it should execute. For example:: @@ -1850,12 +1874,12 @@ Sub-commands >>> args.func(args) ((XYZYX)) - This way, you can let :meth:`parse_args` do the job of calling the + This way, you can let :meth:`~ArgumentParser.parse_args` do the job of calling the appropriate function after argument parsing is complete. Associating functions with actions like this is typically the easiest way to handle the different actions for each of your subparsers. However, if it is necessary to check the name of the subparser that was invoked, the ``dest`` keyword - argument to the :meth:`add_subparsers` call will work:: + argument to the :meth:`~ArgumentParser.add_subparsers` call will work:: >>> parser = argparse.ArgumentParser() >>> subparsers = parser.add_subparsers(dest='subparser_name') @@ -1874,6 +1898,43 @@ Sub-commands the main parser. +.. method:: _SubParsersAction.add_parser(name, *, help=None, aliases=None, \ + deprecated=False, **kwargs) + + Create and return a new :class:`ArgumentParser` object for the + subcommand *name*. + + The *name* argument is the name of the sub-command. + + The *help* argument provides a short description for this sub-command. + + The *aliases* argument allows providing alternative names for this + sub-command. For example:: + + >>> parser = argparse.ArgumentParser() + >>> subparsers = parser.add_subparsers() + >>> checkout = subparsers.add_parser('checkout', aliases=['co']) + >>> checkout.add_argument('foo') + >>> parser.parse_args(['co', 'bar']) + Namespace(foo='bar') + + The *deprecated* argument, if ``True``, marks the sub-command as + deprecated and will issue a warning when used. For example:: + + >>> parser = argparse.ArgumentParser(prog='chicken.py') + >>> subparsers = parser.add_subparsers() + >>> fly = subparsers.add_parser('fly', deprecated=True) + >>> args = parser.parse_args(['fly']) + chicken.py: warning: command 'fly' is deprecated + Namespace() + + All other keyword arguments are passed directly to the + :class:`!ArgumentParser` constructor. + + .. versionadded:: 3.13 + Added the *deprecated* parameter. + + FileType objects ^^^^^^^^^^^^^^^^ @@ -1909,7 +1970,7 @@ FileType objects run and then use the :keyword:`with`-statement to manage the files. .. versionchanged:: 3.4 - Added the *encodings* and *errors* parameters. + Added the *encoding* and *errors* parameters. .. deprecated:: 3.14 @@ -1971,6 +2032,9 @@ Argument groups Note that any arguments not in your user-defined groups will end up back in the usual "positional arguments" and "optional arguments" sections. + Within each argument group, arguments are displayed in help output in the + order in which they are added. + .. deprecated-removed:: 3.11 3.14 Calling :meth:`add_argument_group` on an argument group now raises an exception. This nesting was never supported, often failed to work @@ -2061,7 +2125,9 @@ Parser defaults >>> parser.parse_args(['736']) Namespace(bar=42, baz='badger', foo=736) - Note that parser-level defaults always override argument-level defaults:: + Note that defaults can be set at both the parser level using :meth:`set_defaults` + and at the argument level using :meth:`add_argument`. If both are called for the + same argument, the last default set for an argument is used:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', default='bar') diff --git a/Doc/library/array.rst b/Doc/library/array.rst index 1f04f697c75..4468edb6efa 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -9,7 +9,7 @@ -------------- This module defines an object type which can compactly represent an array of -basic values: characters, integers, floating-point numbers. Arrays are sequence +basic values: characters, integers, floating-point numbers, complex numbers. Arrays are mutable :term:`sequence` types and behave very much like lists, except that the type of objects stored in them is constrained. The type is specified at object creation time by using a :dfn:`type code`, which is a single character. The following type codes are @@ -42,10 +42,17 @@ defined: +-----------+--------------------+-------------------+-----------------------+-------+ | ``'Q'`` | unsigned long long | int | 8 | | +-----------+--------------------+-------------------+-----------------------+-------+ +| ``'e'`` | _Float16 | float | 2 | \(3) | ++-----------+--------------------+-------------------+-----------------------+-------+ | ``'f'`` | float | float | 4 | | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'d'`` | double | float | 8 | | +-----------+--------------------+-------------------+-----------------------+-------+ +| ``'F'`` | float complex | complex | 8 | \(4) | ++-----------+--------------------+-------------------+-----------------------+-------+ +| ``'D'`` | double complex | complex | 16 | \(4) | ++-----------+--------------------+-------------------+-----------------------+-------+ + Notes: @@ -63,6 +70,31 @@ Notes: (2) .. versionadded:: 3.13 +(3) + The IEEE 754 binary16 "half precision" type was introduced in the 2008 + revision of the `IEEE 754 standard `_. + This type is not widely supported by C compilers. It's available + as :c:expr:`_Float16` type, if the compiler supports the Annex H + of the C23 standard. + + .. versionadded:: 3.15 + +(4) + Complex types (``F`` and ``D``) are available unconditionally, + regardless on support for complex types (the Annex G of the C11 standard) + by the C compiler. + As specified in the C11 standard, each complex type is represented by a + two-element C array containing, respectively, the real and imaginary parts. + + .. versionadded:: 3.15 + +.. seealso:: + + The :ref:`ctypes ` and + :ref:`struct ` modules, + as well as third-party modules like `numpy `__, + use similar -- but slightly different -- type codes. + The actual representation of values is determined by the machine architecture (strictly speaking, by the C implementation). The actual size can be accessed @@ -93,7 +125,7 @@ The module defines the following type: otherwise, the initializer's iterator is passed to the :meth:`extend` method to add initial items to the array. - Array objects support the ordinary sequence operations of indexing, slicing, + Array objects support the ordinary :ref:`mutable ` :term:`sequence` operations of indexing, slicing, concatenation, and multiplication. When using slice assignment, the assigned value must be an array object with the same type code; in all other cases, :exc:`TypeError` is raised. Array objects also implement the buffer interface, @@ -112,9 +144,9 @@ The module defines the following type: The length in bytes of one array item in the internal representation. - .. method:: append(x) + .. method:: append(value, /) - Append a new item with value *x* to the end of the array. + Append a new item with the specified value to the end of the array. .. method:: buffer_info() @@ -139,17 +171,18 @@ The module defines the following type: .. method:: byteswap() "Byteswap" all items of the array. This is only supported for values which are - 1, 2, 4, or 8 bytes in size; for other types of values, :exc:`RuntimeError` is + 1, 2, 4, 8 or 16 bytes in size; for other types of values, :exc:`RuntimeError` is raised. It is useful when reading data from a file written on a machine with a - different byte order. + different byte order. Note, that for complex types the order of + components (the real part, followed by imaginary part) is preserved. - .. method:: count(x) + .. method:: count(value, /) - Return the number of occurrences of *x* in the array. + Return the number of occurrences of *value* in the array. - .. method:: extend(iterable) + .. method:: extend(iterable, /) Append items from *iterable* to the end of the array. If *iterable* is another array, it must have *exactly* the same type code; if not, :exc:`TypeError` will @@ -157,7 +190,7 @@ The module defines the following type: must be the right type to be appended to the array. - .. method:: frombytes(buffer) + .. method:: frombytes(buffer, /) Appends items from the :term:`bytes-like object`, interpreting its content as an array of machine values (as if it had been read @@ -167,7 +200,7 @@ The module defines the following type: :meth:`!fromstring` is renamed to :meth:`frombytes` for clarity. - .. method:: fromfile(f, n) + .. method:: fromfile(f, n, /) Read *n* items (as machine values) from the :term:`file object` *f* and append them to the end of the array. If less than *n* items are available, @@ -175,13 +208,13 @@ The module defines the following type: inserted into the array. - .. method:: fromlist(list) + .. method:: fromlist(list, /) Append items from the list. This is equivalent to ``for x in list: a.append(x)`` except that if there is a type error, the array is unchanged. - .. method:: fromunicode(s) + .. method:: fromunicode(ustr, /) Extends this array with data from the given Unicode string. The array must have type code ``'u'`` or ``'w'``; otherwise a :exc:`ValueError` is raised. @@ -189,33 +222,33 @@ The module defines the following type: array of some other type. - .. method:: index(x[, start[, stop]]) + .. method:: index(value[, start[, stop]]) Return the smallest *i* such that *i* is the index of the first occurrence of - *x* in the array. The optional arguments *start* and *stop* can be - specified to search for *x* within a subsection of the array. Raise - :exc:`ValueError` if *x* is not found. + *value* in the array. The optional arguments *start* and *stop* can be + specified to search for *value* within a subsection of the array. Raise + :exc:`ValueError` if *value* is not found. .. versionchanged:: 3.10 Added optional *start* and *stop* parameters. - .. method:: insert(i, x) + .. method:: insert(index, value, /) - Insert a new item with value *x* in the array before position *i*. Negative + Insert a new item *value* in the array before position *index*. Negative values are treated as being relative to the end of the array. - .. method:: pop([i]) + .. method:: pop(index=-1, /) Removes the item with the index *i* from the array and returns it. The optional argument defaults to ``-1``, so that by default the last item is removed and returned. - .. method:: remove(x) + .. method:: remove(value, /) - Remove the first occurrence of *x* from the array. + Remove the first occurrence of *value* from the array. .. method:: clear() @@ -240,7 +273,7 @@ The module defines the following type: :meth:`!tostring` is renamed to :meth:`tobytes` for clarity. - .. method:: tofile(f) + .. method:: tofile(f, /) Write all items (as machine values) to the :term:`file object` *f*. @@ -282,3 +315,5 @@ Examples:: `NumPy `_ The NumPy package defines another array type. + +.. _ieee 754 standard: https://en.wikipedia.org/wiki/IEEE_754-2008_revision diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 319b2c81505..e10b8e22df0 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -4,9 +4,6 @@ .. module:: ast :synopsis: Abstract Syntax Tree classes and manipulation. -.. sectionauthor:: Martin v. Löwis -.. sectionauthor:: Georg Brandl - .. testsetup:: import ast @@ -15,7 +12,7 @@ -------------- -The :mod:`ast` module helps Python applications to process trees of the Python +The :mod:`!ast` module helps Python applications to process trees of the Python abstract syntax grammar. The abstract syntax itself might change with each Python release; this module helps to find out programmatically what the current grammar looks like. @@ -38,15 +35,17 @@ The abstract grammar is currently defined as follows: :language: asdl +.. _ast_nodes: + Node classes ------------ .. class:: AST - This is the base of all AST node classes. The actual node classes are + This is the abstract base of all AST node classes. The actual node classes are derived from the :file:`Parser/Python.asdl` file, which is reproduced :ref:`above `. They are defined in the :mod:`!_ast` C - module and re-exported in :mod:`ast`. + module and re-exported in :mod:`!ast`. There is one class defined for each left-hand side symbol in the abstract grammar (for example, :class:`ast.stmt` or :class:`ast.expr`). In addition, @@ -134,17 +133,26 @@ Node classes Simple indices are represented by their value, extended slices are represented as tuples. +.. versionchanged:: 3.13 + + AST node constructors were changed to provide sensible defaults for omitted + fields: optional fields now default to ``None``, list fields default to an + empty list, and fields of type :class:`!ast.expr_context` default to + :class:`Load() `. Previously, omitted attributes would not exist on constructed + nodes (accessing them raised :exc:`AttributeError`). + .. versionchanged:: 3.14 The :meth:`~object.__repr__` output of :class:`~ast.AST` nodes includes the values of the node fields. -.. deprecated:: 3.8 +.. deprecated-removed:: 3.8 3.14 - Old classes :class:`!ast.Num`, :class:`!ast.Str`, :class:`!ast.Bytes`, - :class:`!ast.NameConstant` and :class:`!ast.Ellipsis` are still available, - but they will be removed in future Python releases. In the meantime, - instantiating them will return an instance of a different class. + Previous versions of Python provided the AST classes :class:`!ast.Num`, + :class:`!ast.Str`, :class:`!ast.Bytes`, :class:`!ast.NameConstant` and + :class:`!ast.Ellipsis`, which were deprecated in Python 3.8. These classes + were removed in Python 3.14, and their functionality has been replaced with + :class:`ast.Constant`. .. deprecated:: 3.9 @@ -158,8 +166,16 @@ Node classes Previous versions of Python allowed the creation of AST nodes that were missing required fields. Similarly, AST node constructors allowed arbitrary keyword arguments that were set as attributes of the AST node, even if they did not - match any of the fields of the AST node. This behavior is deprecated and will - be removed in Python 3.15. + match any of the fields of the AST node. These cases now raise a :exc:`TypeError`. + +.. deprecated-removed:: next 3.20 + + In the :ref:`grammar above `, the AST node classes that + correspond to production rules with variants (aka "sums") are abstract + classes. Previous versions of Python allowed for the creation of direct + instances of these abstract node classes. This behavior is deprecated and + will be removed in Python 3.20. + .. note:: The descriptions of the specific node classes displayed here @@ -264,18 +280,25 @@ Root nodes Literals ^^^^^^^^ -.. class:: Constant(value) +.. class:: Constant(value, kind) A constant value. The ``value`` attribute of the ``Constant`` literal contains the Python object it represents. The values represented can be instances of :class:`str`, :class:`bytes`, :class:`int`, :class:`float`, :class:`complex`, and :class:`bool`, and the constants :data:`None` and :data:`Ellipsis`. + The ``kind`` attribute is an optional string. For string literals with a + ``u`` prefix, ``kind`` is set to ``'u'``. For all other + constants, ``kind`` is ``None``. + .. doctest:: >>> print(ast.dump(ast.parse('123', mode='eval'), indent=4)) Expression( body=Constant(value=123)) + >>> print(ast.dump(ast.parse("u'hello'", mode='eval'), indent=4)) + Expression( + body=Constant(value='hello', kind='u')) .. class:: FormattedValue(value, conversion, format_spec) @@ -363,6 +386,11 @@ Literals function call). This has the same meaning as ``FormattedValue.value``. * ``str`` is a constant containing the text of the interpolation expression. + + If ``str`` is set to ``None``, then ``value`` is used to generate code + when calling :func:`ast.unparse`. This no longer guarantees that the + generated code is identical to the original and is intended for code + generation. * ``conversion`` is an integer: * -1: no conversion @@ -1108,7 +1136,8 @@ Imports names=[ alias(name='x'), alias(name='y'), - alias(name='z')])]) + alias(name='z')], + is_lazy=0)]) .. class:: ImportFrom(module, names, level) @@ -1129,7 +1158,8 @@ Imports alias(name='x'), alias(name='y'), alias(name='z')], - level=0)]) + level=0, + is_lazy=0)]) .. class:: alias(name, asname) @@ -1147,7 +1177,8 @@ Imports names=[ alias(name='a', asname='b'), alias(name='c')], - level=2)]) + level=2, + is_lazy=0)]) Control flow ^^^^^^^^^^^^ @@ -2194,16 +2225,16 @@ Async and await occurrences of the same value (for example, :class:`ast.Add`). -:mod:`ast` helpers ------------------- +:mod:`!ast` helpers +------------------- -Apart from the node classes, the :mod:`ast` module defines these utility functions +Apart from the node classes, the :mod:`!ast` module defines these utility functions and classes for traversing abstract syntax trees: -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1) +.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1, module=None) Parse the source into an AST node. Equivalent to ``compile(source, - filename, mode, flags=FLAGS_VALUE, optimize=optimize)``, + filename, mode, flags=FLAGS_VALUE, optimize=optimize, module=module)``, where ``FLAGS_VALUE`` is ``ast.PyCF_ONLY_AST`` if ``optimize <= 0`` and ``ast.PyCF_OPTIMIZED_AST`` otherwise. @@ -2256,6 +2287,9 @@ and classes for traversing abstract syntax trees: The minimum supported version for ``feature_version`` is now ``(3, 7)``. The ``optimize`` argument was added. + .. versionadded:: 3.15 + Added the *module* parameter. + .. function:: unparse(ast_obj) @@ -2411,12 +2445,12 @@ and classes for traversing abstract syntax trees: during traversal. For this a special visitor exists (:class:`NodeTransformer`) that allows modifications. - .. deprecated:: 3.8 + .. deprecated-removed:: 3.8 3.14 Methods :meth:`!visit_Num`, :meth:`!visit_Str`, :meth:`!visit_Bytes`, - :meth:`!visit_NameConstant` and :meth:`!visit_Ellipsis` are deprecated - now and will not be called in future Python versions. Add the - :meth:`visit_Constant` method to handle all constant nodes. + :meth:`!visit_NameConstant` and :meth:`!visit_Ellipsis` will not be called + in Python 3.14+. Add the :meth:`visit_Constant` method instead to handle + all constant nodes. .. class:: NodeTransformer() @@ -2463,7 +2497,7 @@ and classes for traversing abstract syntax trees: node = YourTransformer().visit(node) -.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None, show_empty=False) +.. function:: dump(node, annotate_fields=True, include_attributes=False, *, color=False, indent=None, show_empty=False) Return a formatted dump of the tree in *node*. This is mainly useful for debugging purposes. If *annotate_fields* is true (by default), @@ -2473,6 +2507,10 @@ and classes for traversing abstract syntax trees: numbers and column offsets are not dumped by default. If this is wanted, *include_attributes* can be set to true. + If *color* is ``True``, the returned string is syntax highlighted using + ANSI escape sequences. + If ``False`` (the default), colored output is always disabled. + If *indent* is a non-negative integer or string, then the tree will be pretty-printed with that indent level. An indent level of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) @@ -2507,9 +2545,26 @@ and classes for traversing abstract syntax trees: .. versionchanged:: 3.13 Added the *show_empty* option. - .. versionchanged:: next + .. versionchanged:: 3.15 Omit optional ``Load()`` values by default. + .. versionchanged:: next + Added the *color* parameter. + + +.. function:: compare(a, b, /, *, compare_attributes=False) + + Recursively compares two ASTs. + + *compare_attributes* affects whether AST attributes are considered + in the comparison. If *compare_attributes* is ``False`` (default), then + attributes are ignored. Otherwise they must all be equal. This + option is useful to check whether the ASTs are structurally equal but + differ in whitespace or similar details. Attributes include line numbers + and column offsets. + + .. versionadded:: 3.14 + .. _ast-compiler-flags: @@ -2546,20 +2601,6 @@ effects on the compilation of a program: .. versionadded:: 3.8 -.. function:: compare(a, b, /, *, compare_attributes=False) - - Recursively compares two ASTs. - - *compare_attributes* affects whether AST attributes are considered - in the comparison. If *compare_attributes* is ``False`` (default), then - attributes are ignored. Otherwise they must all be equal. This - option is useful to check whether the ASTs are structurally equal but - differ in whitespace or similar details. Attributes include line numbers - and column offsets. - - .. versionadded:: 3.14 - - .. _ast-cli: Command-line usage @@ -2567,7 +2608,11 @@ Command-line usage .. versionadded:: 3.9 -The :mod:`ast` module can be executed as a script from the command line. +.. versionchanged:: next + The output is now syntax highlighted by default. This can be + :ref:`controlled using environment variables `. + +The :mod:`!ast` module can be executed as a script from the command line. It is as simple as: .. code-block:: sh diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index 7831b613bd4..713b40d7466 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -248,3 +248,225 @@ Output in debug mode:: File "../t.py", line 4, in bug raise Exception("not consumed") Exception: not consumed + + +Asynchronous generators best practices +====================================== + +Writing correct and efficient asyncio code requires awareness of certain pitfalls. +This section outlines essential best practices that can save you hours of debugging. + + +Close asynchronous generators explicitly +---------------------------------------- + +It is recommended to manually close the +:term:`asynchronous generator `. If a generator +exits early - for example, due to an exception raised in the body of +an ``async for`` loop - its asynchronous cleanup code may run in an +unexpected context. This can occur after the tasks it depends on have completed, +or during the event loop shutdown when the async-generator's garbage collection +hook is called. + +To avoid this, explicitly close the generator by calling its +:meth:`~agen.aclose` method, or use the :func:`contextlib.aclosing` +context manager:: + + import asyncio + import contextlib + + async def gen(): + yield 1 + yield 2 + + async def func(): + async with contextlib.aclosing(gen()) as g: + async for x in g: + break # Don't iterate until the end + + asyncio.run(func()) + +As noted above, the cleanup code for these asynchronous generators is deferred. +The following example demonstrates that the finalization of an asynchronous +generator can occur in an unexpected order:: + + import asyncio + work_done = False + + async def cursor(): + try: + yield 1 + finally: + assert work_done + + async def rows(): + global work_done + try: + yield 2 + finally: + await asyncio.sleep(0.1) # imitate some async work + work_done = True + + + async def main(): + async for c in cursor(): + async for r in rows(): + break + break + + asyncio.run(main()) + +For this example, we get the following output:: + + unhandled exception during asyncio.run() shutdown + task: ()> exception=AssertionError()> + Traceback (most recent call last): + File "example.py", line 6, in cursor + yield 1 + asyncio.exceptions.CancelledError + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "example.py", line 8, in cursor + assert work_done + ^^^^^^^^^ + AssertionError + +The ``cursor()`` asynchronous generator was finalized before the ``rows`` +generator - an unexpected behavior. + +The example can be fixed by explicitly closing the +``cursor`` and ``rows`` async-generators:: + + async def main(): + async with contextlib.aclosing(cursor()) as cursor_gen: + async for c in cursor_gen: + async with contextlib.aclosing(rows()) as rows_gen: + async for r in rows_gen: + break + break + + +Create asynchronous generators only when the event loop is running +------------------------------------------------------------------ + +It is recommended to create +:term:`asynchronous generators ` only after +the event loop has been created. + +To ensure that asynchronous generators close reliably, the event loop uses the +:func:`sys.set_asyncgen_hooks` function to register callback functions. These +callbacks update the list of running asynchronous generators to keep it in a +consistent state. + +When the :meth:`loop.shutdown_asyncgens() ` +function is called, the running generators are stopped gracefully and the +list is cleared. + +The asynchronous generator invokes the corresponding system hook during its +first iteration. At the same time, the generator records that the hook has +been called and does not call it again. + +Therefore, if iteration begins before the event loop is created, +the event loop will not be able to add the generator to its list of active +generators because the hooks are set after the generator attempts to call them. +Consequently, the event loop will not be able to terminate the generator +if necessary. + +Consider the following example:: + + import asyncio + + async def agenfn(): + try: + yield 10 + finally: + await asyncio.sleep(0) + + + with asyncio.Runner() as runner: + agen = agenfn() + print(runner.run(anext(agen))) + del agen + +Output:: + + 10 + Exception ignored while closing generator : + Traceback (most recent call last): + File "example.py", line 13, in + del agen + ^^^^ + RuntimeError: async generator ignored GeneratorExit + +This example can be fixed as follows:: + + import asyncio + + async def agenfn(): + try: + yield 10 + finally: + await asyncio.sleep(0) + + async def main(): + agen = agenfn() + print(await anext(agen)) + del agen + + asyncio.run(main()) + + +Avoid concurrent iteration and closure of the same generator +------------------------------------------------------------ + +Async generators may be reentered while another +:meth:`~agen.__anext__` / :meth:`~agen.athrow` / :meth:`~agen.aclose` call is in +progress. This may lead to an inconsistent state of the async generator and can +cause errors. + +Let's consider the following example:: + + import asyncio + + async def consumer(): + for idx in range(100): + await asyncio.sleep(0) + message = yield idx + print('received', message) + + async def amain(): + agenerator = consumer() + await agenerator.asend(None) + + fa = asyncio.create_task(agenerator.asend('A')) + fb = asyncio.create_task(agenerator.asend('B')) + await fa + await fb + + asyncio.run(amain()) + +Output:: + + received A + Traceback (most recent call last): + File "test.py", line 38, in + asyncio.run(amain()) + ~~~~~~~~~~~^^^^^^^^^ + File "Lib/asyncio/runners.py", line 204, in run + return runner.run(main) + ~~~~~~~~~~^^^^^^ + File "Lib/asyncio/runners.py", line 127, in run + return self._loop.run_until_complete(task) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ + File "Lib/asyncio/base_events.py", line 719, in run_until_complete + return future.result() + ~~~~~~~~~~~~~^^ + File "test.py", line 36, in amain + await fb + RuntimeError: anext(): asynchronous generator is already running + + +Therefore, it is recommended to avoid using asynchronous generators in parallel +tasks or across multiple event loops. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 0ccc7a2b448..79c9516cda2 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -4,7 +4,7 @@ .. _asyncio-event-loop: ========== -Event Loop +Event loop ========== **Source code:** :source:`Lib/asyncio/events.py`, @@ -105,7 +105,7 @@ This documentation page contains the following sections: .. _asyncio-event-loop-methods: -Event Loop Methods +Event loop methods ================== Event loops have **low-level** APIs for the following: @@ -297,8 +297,9 @@ clocks to track time. are called is undefined. The optional positional *args* will be passed to the callback when - it is called. If you want the callback to be called with keyword - arguments use :func:`functools.partial`. + it is called. Use :func:`functools.partial` + :ref:`to pass keyword arguments ` to + *callback*. An optional keyword-only *context* argument allows specifying a custom :class:`contextvars.Context` for the *callback* to run in. @@ -360,7 +361,7 @@ clocks to track time. The :func:`asyncio.sleep` function. -Creating Futures and Tasks +Creating futures and tasks ^^^^^^^^^^^^^^^^^^^^^^^^^^ .. method:: loop.create_future() @@ -961,7 +962,7 @@ Transferring files .. versionadded:: 3.7 -TLS Upgrade +TLS upgrade ^^^^^^^^^^^ .. method:: loop.start_tls(transport, protocol, \ @@ -1034,8 +1035,8 @@ Watching file descriptors .. method:: loop.add_writer(fd, callback, *args) Start monitoring the *fd* file descriptor for write availability and - invoke *callback* with the specified arguments once *fd* is available for - writing. + invoke *callback* with the specified arguments *args* once *fd* is + available for writing. Any preexisting callback registered for *fd* is cancelled and replaced by *callback*. @@ -1308,7 +1309,8 @@ Unix signals .. method:: loop.add_signal_handler(signum, callback, *args) - Set *callback* as the handler for the *signum* signal. + Set *callback* as the handler for the *signum* signal, + passing *args* as positional arguments. The callback will be invoked by *loop*, along with other queued callbacks and runnable coroutines of that event loop. Unlike signal handlers @@ -1343,7 +1345,8 @@ Executing code in thread or process pools .. awaitablemethod:: loop.run_in_executor(executor, func, *args) - Arrange for *func* to be called in the specified executor. + Arrange for *func* to be called in the specified executor + passing *args* as positional arguments. The *executor* argument should be an :class:`concurrent.futures.Executor` instance. The default executor is used if *executor* is ``None``. @@ -1428,7 +1431,7 @@ Executing code in thread or process pools :class:`~concurrent.futures.ThreadPoolExecutor`. -Error Handling API +Error handling API ^^^^^^^^^^^^^^^^^^ Allows customizing how exceptions are handled in the event loop. @@ -1531,7 +1534,7 @@ Enabling debug mode The :ref:`debug mode of asyncio `. -Running Subprocesses +Running subprocesses ^^^^^^^^^^^^^^^^^^^^ Methods described in this subsections are low-level. In regular @@ -1631,6 +1634,9 @@ async/await code consider using the high-level conforms to the :class:`asyncio.SubprocessTransport` base class and *protocol* is an object instantiated by the *protocol_factory*. + If the transport is closed or is garbage collected, the child process + is killed if it is still running. + .. method:: loop.subprocess_shell(protocol_factory, cmd, *, \ stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ stderr=subprocess.PIPE, **kwargs) @@ -1654,6 +1660,9 @@ async/await code consider using the high-level conforms to the :class:`SubprocessTransport` base class and *protocol* is an object instantiated by the *protocol_factory*. + If the transport is closed or is garbage collected, the child process + is killed if it is still running. + .. note:: It is the application's responsibility to ensure that all whitespace and special characters are quoted appropriately to avoid `shell injection @@ -1663,7 +1672,7 @@ async/await code consider using the high-level are going to be used to construct shell commands. -Callback Handles +Callback handles ================ .. class:: Handle @@ -1706,7 +1715,7 @@ Callback Handles .. versionadded:: 3.7 -Server Objects +Server objects ============== Server objects are created by :meth:`loop.create_server`, @@ -1849,7 +1858,7 @@ Do not instantiate the :class:`Server` class directly. .. _asyncio-event-loops: .. _asyncio-event-loop-implementations: -Event Loop Implementations +Event loop implementations ========================== asyncio ships with two different event loop implementations: @@ -1962,10 +1971,10 @@ callback uses the :meth:`loop.call_later` method to reschedule itself after 5 seconds, and then stops the event loop:: import asyncio - import datetime + import datetime as dt def display_date(end_time, loop): - print(datetime.datetime.now()) + print(dt.datetime.now()) if (loop.time() + 1.0) < end_time: loop.call_later(1, display_date, end_time, loop) else: @@ -2046,7 +2055,7 @@ Wait until a file descriptor received some data using the Set signal handlers for SIGINT and SIGTERM ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -(This ``signals`` example only works on Unix.) +(This ``signal`` example only works on Unix.) Register handlers for signals :const:`~signal.SIGINT` and :const:`~signal.SIGTERM` using the :meth:`loop.add_signal_handler` method:: diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 4b69e569523..43977de273e 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -196,6 +196,10 @@ Future Object Otherwise, change the Future's state to *cancelled*, schedule the callbacks, and return ``True``. + The optional string argument *msg* is passed as the argument to the + :exc:`CancelledError` exception raised when a cancelled Future + is awaited. + .. versionchanged:: 3.9 Added the *msg* parameter. diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 5208f14c94a..58f77feb311 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -1037,7 +1037,7 @@ The subprocess is created by the :meth:`loop.subprocess_exec` method:: # low-level APIs. loop = asyncio.get_running_loop() - code = 'import datetime; print(datetime.datetime.now())' + code = 'import datetime as dt; print(dt.datetime.now())' exit_future = asyncio.Future(loop=loop) # Create the subprocess controlled by DateProtocol; diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index d481a1921d5..a9735ae8065 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -107,7 +107,7 @@ Queue The queue can no longer grow. Future calls to :meth:`~Queue.put` raise :exc:`QueueShutDown`. Currently blocked callers of :meth:`~Queue.put` will be unblocked - and will raise :exc:`QueueShutDown` in the formerly blocked thread. + and will raise :exc:`QueueShutDown` in the formerly awaiting task. If *immediate* is false (the default), the queue can be wound down normally with :meth:`~Queue.get` calls to extract tasks diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index e1568ae330b..05445219510 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -316,13 +316,14 @@ StreamWriter If that fails, the data is queued in an internal write buffer until it can be sent. + The *data* buffer should be a bytes, bytearray, or C-contiguous one-dimensional + memoryview object. + The method should be used along with the ``drain()`` method:: stream.write(data) await stream.drain() - .. note:: - The *data* buffer should be a C contiguous one-dimensional :term:`bytes-like object `. .. method:: writelines(data) diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 03e76bc8689..a6514649bf9 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -76,6 +76,9 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_exec` for other parameters. + If the process object is garbage collected while the process is still + running, the child process will be killed. + .. versionchanged:: 3.10 Removed the *loop* parameter. @@ -95,6 +98,9 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_shell` for other parameters. + If the process object is garbage collected while the process is still + running, the child process will be killed. + .. important:: It is the application's responsibility to ensure that all whitespace and @@ -305,8 +311,16 @@ their completion. A ``None`` value indicates that the process has not terminated yet. - A negative value ``-N`` indicates that the child was terminated - by signal ``N`` (POSIX only). + For processes created with :func:`~asyncio.create_subprocess_exec`, a negative + value ``-N`` indicates that the child was terminated by signal ``N`` + (POSIX only). + + For processes created with :func:`~asyncio.create_subprocess_shell`, the + return code reflects the exit status of the shell itself (e.g. ``/bin/sh``), + which may map signals to codes such as ``128+N``. See the + documentation of the shell (for example, the Bash manual's Exit Status) + for details. + .. _asyncio-subprocess-threads: @@ -345,7 +359,7 @@ function:: import sys async def get_date(): - code = 'import datetime; print(datetime.datetime.now())' + code = 'import datetime as dt; print(dt.datetime.now())' # Create the subprocess; redirect the standard output # into a pipe. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index f825ae92ec7..2e17d0dc70c 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -2,7 +2,7 @@ ==================== -Coroutines and Tasks +Coroutines and tasks ==================== This section outlines high-level asyncio APIs to work with coroutines @@ -231,7 +231,7 @@ A good example of a low-level function that returns a Future object is :meth:`loop.run_in_executor`. -Creating Tasks +Creating tasks ============== **Source code:** :source:`Lib/asyncio/tasks.py` @@ -300,7 +300,7 @@ Creating Tasks Added the *eager_start* parameter by passing on all *kwargs*. -Task Cancellation +Task cancellation ================= Tasks can easily and safely be cancelled. @@ -324,7 +324,7 @@ remove the cancellation state. .. _taskgroups: -Task Groups +Task groups =========== Task groups combine a task creation API with a convenient @@ -355,6 +355,34 @@ and reliable way to wait for all tasks in the group to finish. Passes on all *kwargs* to :meth:`loop.create_task` + .. method:: cancel() + + Cancel the task group. This is a non-exceptional, early exit of the + task group's lifetime -- useful once the group's goal has been met or + its services no longer needed. + + :meth:`~asyncio.Task.cancel` will be called on any tasks in the group that + aren't yet done, as well as the parent (body) of the group. The task group + context manager will exit *without* :exc:`asyncio.CancelledError` being raised. + + If :meth:`cancel` is called before entering the task group, the group will be + cancelled upon entry. This is useful for patterns where one piece of + code passes an unused :class:`asyncio.TaskGroup` instance to another in order to have + the ability to cancel anything run within the group. + + :meth:`cancel` is idempotent and may be called after the task group has + already exited. + + Some ways to use :meth:`cancel`: + + * call it from the task group body based on some condition or event + * pass the task group instance to child tasks via :meth:`create_task`, allowing a child + task to conditionally cancel the entire entire group + * pass the task group instance or bound :meth:`cancel` method to some other task *before* + opening the task group, allowing remote cancellation + + .. versionadded:: next + Example:: async def main(): @@ -366,7 +394,8 @@ Example:: The ``async with`` statement will wait for all tasks in the group to finish. While waiting, new tasks may still be added to the group (for example, by passing ``tg`` into one of the coroutines -and calling ``tg.create_task()`` in that coroutine). +and calling ``tg.create_task()`` in that coroutine). There is also opportunity to +request termination of the entire task group with ``tg.cancel()``, based on some condition. Once the last task has finished and the ``async with`` block is exited, no new tasks may be added to the group. @@ -427,53 +456,6 @@ reported by :meth:`asyncio.Task.cancelling`. Improved handling of simultaneous internal and external cancellations and correct preservation of cancellation counts. -Terminating a Task Group ------------------------- - -While terminating a task group is not natively supported by the standard -library, termination can be achieved by adding an exception-raising task -to the task group and ignoring the raised exception: - -.. code-block:: python - - import asyncio - from asyncio import TaskGroup - - class TerminateTaskGroup(Exception): - """Exception raised to terminate a task group.""" - - async def force_terminate_task_group(): - """Used to force termination of a task group.""" - raise TerminateTaskGroup() - - async def job(task_id, sleep_time): - print(f'Task {task_id}: start') - await asyncio.sleep(sleep_time) - print(f'Task {task_id}: done') - - async def main(): - try: - async with TaskGroup() as group: - # spawn some tasks - group.create_task(job(1, 0.5)) - group.create_task(job(2, 1.5)) - # sleep for 1 second - await asyncio.sleep(1) - # add an exception-raising task to force the group to terminate - group.create_task(force_terminate_task_group()) - except* TerminateTaskGroup: - pass - - asyncio.run(main()) - -Expected output: - -.. code-block:: text - - Task 1: start - Task 2: start - Task 1: done - Sleeping ======== @@ -498,13 +480,13 @@ Sleeping for 5 seconds:: import asyncio - import datetime + import datetime as dt async def display_date(): loop = asyncio.get_running_loop() end_time = loop.time() + 5.0 while True: - print(datetime.datetime.now()) + print(dt.datetime.now()) if (loop.time() + 1.0) >= end_time: break await asyncio.sleep(1) @@ -519,7 +501,7 @@ Sleeping Raises :exc:`ValueError` if *delay* is :data:`~math.nan`. -Running Tasks Concurrently +Running tasks concurrently ========================== .. awaitablefunction:: gather(*aws, return_exceptions=False) @@ -557,7 +539,7 @@ Running Tasks Concurrently provides stronger safety guarantees than *gather* for scheduling a nesting of subtasks: if a task (or a subtask, a task scheduled by a task) raises an exception, *TaskGroup* will, while *gather* will not, - cancel the remaining scheduled tasks). + cancel the remaining scheduled tasks. .. _asyncio_example_gather: @@ -621,7 +603,7 @@ Running Tasks Concurrently .. _eager-task-factory: -Eager Task Factory +Eager task factory ================== .. function:: eager_task_factory(loop, coro, *, name=None, context=None) @@ -664,7 +646,7 @@ Eager Task Factory .. versionadded:: 3.12 -Shielding From Cancellation +Shielding from cancellation =========================== .. awaitablefunction:: shield(aw) @@ -771,6 +753,9 @@ Timeouts An :ref:`asynchronous context manager ` for cancelling overdue coroutines. + Prefer using :func:`asyncio.timeout` or :func:`asyncio.timeout_at` + rather than instantiating :class:`!Timeout` directly. + ``when`` should be an absolute time at which the context should time out, as measured by the event loop's clock: @@ -891,7 +876,7 @@ Timeouts Raises :exc:`TimeoutError` instead of :exc:`asyncio.TimeoutError`. -Waiting Primitives +Waiting primitives ================== .. function:: wait(aws, *, timeout=None, return_when=ALL_COMPLETED) @@ -1011,7 +996,7 @@ Waiting Primitives or as a plain :term:`iterator` (previously it was only a plain iterator). -Running in Threads +Running in threads ================== .. function:: to_thread(func, /, *args, **kwargs) @@ -1071,7 +1056,7 @@ Running in Threads .. versionadded:: 3.9 -Scheduling From Other Threads +Scheduling from other threads ============================= .. function:: run_coroutine_threadsafe(coro, loop) @@ -1195,7 +1180,7 @@ Introspection .. _asyncio-task-obj: -Task Object +Task object =========== .. class:: Task(coro, *, loop=None, name=None, context=None, eager_start=False) @@ -1221,8 +1206,8 @@ Task Object To cancel a running Task use the :meth:`cancel` method. Calling it will cause the Task to throw a :exc:`CancelledError` exception into - the wrapped coroutine. If a coroutine is awaiting on a Future - object during cancellation, the Future object will be cancelled. + the wrapped coroutine. If a coroutine is awaiting on a future-like + object during cancellation, the awaited object will be cancelled. :meth:`cancelled` can be used to check if the Task was cancelled. The method returns ``True`` if the wrapped coroutine did not @@ -1411,6 +1396,10 @@ Task Object the cancellation, it needs to call :meth:`Task.uncancel` in addition to catching the exception. + If the Task being cancelled is currently awaiting on a future-like + object, that awaited object will also be cancelled. This cancellation + propagates down the entire chain of awaited objects. + .. versionchanged:: 3.9 Added the *msg* parameter. diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 444db01390d..0f72e31dee5 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -79,6 +79,10 @@ You can experiment with an ``asyncio`` concurrent context in the :term:`REPL`: >>> await asyncio.sleep(10, result='hello') 'hello' +This REPL provides limited compatibility with :envvar:`PYTHON_BASIC_REPL`. +It is recommended that the default REPL is used +for full functionality and the latest features. + .. audit-event:: cpython.run_stdin "" "" .. versionchanged:: 3.12.5 (also 3.11.10, 3.10.15, 3.9.20, and 3.8.20) diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index 02d2f0807df..b5caf5502d0 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -4,14 +4,11 @@ .. module:: atexit :synopsis: Register and execute cleanup functions. -.. moduleauthor:: Skip Montanaro -.. sectionauthor:: Skip Montanaro - -------------- -The :mod:`atexit` module defines functions to register and unregister cleanup +The :mod:`!atexit` module defines functions to register and unregister cleanup functions. Functions thus registered are automatically executed upon normal -interpreter termination. :mod:`atexit` runs these functions in the *reverse* +interpreter termination. :mod:`!atexit` runs these functions in the *reverse* order in which they were registered; if you register ``A``, ``B``, and ``C``, at interpreter termination time they will be run in the order ``C``, ``B``, ``A``. @@ -64,7 +61,7 @@ a cleanup function is undefined. Remove *func* from the list of functions to be run at interpreter shutdown. :func:`unregister` silently does nothing if *func* was not previously registered. If *func* has been registered more than once, every occurrence - of that function in the :mod:`atexit` call stack will be removed. Equality + of that function in the :mod:`!atexit` call stack will be removed. Equality comparisons (``==``) are used internally during unregistration, so function references do not need to have matching identities. @@ -72,14 +69,14 @@ a cleanup function is undefined. .. seealso:: Module :mod:`readline` - Useful example of :mod:`atexit` to read and write :mod:`readline` history + Useful example of :mod:`!atexit` to read and write :mod:`readline` history files. .. _atexit-example: -:mod:`atexit` Example ---------------------- +:mod:`!atexit` Example +---------------------- The following simple example demonstrates how a module can initialize a counter from a file when it is imported and save the counter's updated value diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 529a7242443..32da8294c5a 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -51,7 +51,7 @@ The :rfc:`4648` encodings are suitable for encoding binary data so that it can b safely sent by email, used as parts of URLs, or included as part of an HTTP POST request. -.. function:: b64encode(s, altchars=None) +.. function:: b64encode(s, altchars=None, *, padded=True, wrapcol=0) Encode the :term:`bytes-like object` *s* using Base64 and return the encoded :class:`bytes`. @@ -61,11 +61,20 @@ POST request. This allows an application to e.g. generate URL or filesystem safe Base64 strings. The default is ``None``, for which the standard Base64 alphabet is used. - May assert or raise a :exc:`ValueError` if the length of *altchars* is not 2. Raises a - :exc:`TypeError` if *altchars* is not a :term:`bytes-like object`. + If *padded* is true (default), pad the encoded data with the '=' + character to a size multiple of 4. + If *padded* is false, do not add the pad characters. + + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. + + .. versionchanged:: 3.15 + Added the *padded* and *wrapcol* parameters. -.. function:: b64decode(s, altchars=None, validate=False) +.. function:: b64decode(s, altchars=None, validate=False, *, padded=True, canonical=False) + b64decode(s, altchars=None, validate=True, *, ignorechars, padded=True, canonical=False) Decode the Base64 encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. @@ -74,18 +83,47 @@ POST request. of length 2 which specifies the alternative alphabet used instead of the ``+`` and ``/`` characters. + If *padded* is true, the last group of 4 base 64 alphabet characters must + be padded with the '=' character. + If *padded* is false, padding is neither required nor recognized: + the '=' character is not treated as padding but as a non-alphabet + character, which means it is silently discarded when *validate* is false, + or causes an :exc:`~binascii.Error` when *validate* is true unless + b'=' is included in *ignorechars*. + A :exc:`binascii.Error` exception is raised if *s* is incorrectly padded. - If *validate* is ``False`` (the default), characters that are neither - in the normal base-64 alphabet nor the alternative alphabet are - discarded prior to the padding check. If *validate* is ``True``, - these non-alphabet characters in the input result in a - :exc:`binascii.Error`. + If *ignorechars* is specified, it should be a :term:`bytes-like object` + containing characters to ignore from the input when *validate* is true. + If *ignorechars* contains the pad character ``'='``, the pad characters + presented before the end of the encoded data and the excess pad characters + will be ignored. + The default value of *validate* is ``True`` if *ignorechars* is specified, + ``False`` otherwise. + + If *validate* is false, characters that are neither + in the normal base-64 alphabet nor (if *ignorechars* is not specified) + the alternative alphabet are + discarded prior to the padding check, but the ``+`` and ``/`` characters + keep their meaning if they are not in *altchars* (they will be discarded + in future Python versions). + + If *validate* is true, these non-alphabet characters in the input + result in a :exc:`binascii.Error`. + + If *canonical* is true, non-zero padding bits are rejected. + See :func:`binascii.a2b_base64` for details. For more information about the strict base64 check, see :func:`binascii.a2b_base64` - May assert or raise a :exc:`ValueError` if the length of *altchars* is not 2. + .. versionchanged:: 3.15 + Added the *canonical*, *ignorechars*, and *padded* parameters. + + .. deprecated:: 3.15 + Accepting the ``+`` and ``/`` characters with an alternative alphabet + is now deprecated. + .. function:: standard_b64encode(s) @@ -99,16 +137,19 @@ POST request. Base64 alphabet and return the decoded :class:`bytes`. -.. function:: urlsafe_b64encode(s) +.. function:: urlsafe_b64encode(s, *, padded=True) Encode :term:`bytes-like object` *s* using the URL- and filesystem-safe alphabet, which substitutes ``-`` instead of ``+`` and ``_`` instead of ``/`` in the standard Base64 alphabet, and return the encoded :class:`bytes`. The result - can still contain ``=``. + can still contain ``=`` if *padded* is true (default). + + .. versionchanged:: 3.15 + Added the *padded* parameter. -.. function:: urlsafe_b64decode(s) +.. function:: urlsafe_b64decode(s, *, padded=False) Decode :term:`bytes-like object` or ASCII string *s* using the URL- and filesystem-safe @@ -116,14 +157,32 @@ POST request. ``/`` in the standard Base64 alphabet, and return the decoded :class:`bytes`. + .. versionchanged:: 3.15 + Added the *padded* parameter. + Padding of input is no longer required by default. -.. function:: b32encode(s) + .. deprecated:: 3.15 + Accepting the ``+`` and ``/`` characters is now deprecated. + + +.. function:: b32encode(s, *, padded=True, wrapcol=0) Encode the :term:`bytes-like object` *s* using Base32 and return the encoded :class:`bytes`. + If *padded* is true (default), pad the encoded data with the '=' + character to a size multiple of 8. + If *padded* is false, do not add the pad characters. -.. function:: b32decode(s, casefold=False, map01=None) + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not add any newlines. + + .. versionchanged:: 3.15 + Added the *padded* and *wrapcol* parameters. + + +.. function:: b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b'', canonical=False) Decode the Base32 encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. @@ -139,20 +198,39 @@ POST request. digit 0 is always mapped to the letter O). For security purposes the default is ``None``, so that 0 and 1 are not allowed in the input. + If *padded* is true, the last group of 8 base 32 alphabet characters must + be padded with the '=' character. + If *padded* is false, padding is neither required nor recognized: + the '=' character is not treated as padding but as a non-alphabet + character, which means it raises an :exc:`~binascii.Error` unless + b'=' is included in *ignorechars*. + + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + + If *canonical* is true, non-zero padding bits are rejected. + See :func:`binascii.a2b_base32` for details. + A :exc:`binascii.Error` is raised if *s* is incorrectly padded or if there are non-alphabet characters present in the input. + .. versionchanged:: 3.15 + Added the *canonical*, *ignorechars*, and *padded* parameters. -.. function:: b32hexencode(s) + +.. function:: b32hexencode(s, *, padded=True, wrapcol=0) Similar to :func:`b32encode` but uses the Extended Hex Alphabet, as defined in :rfc:`4648`. .. versionadded:: 3.10 + .. versionchanged:: 3.15 + Added the *padded* and *wrapcol* parameters. -.. function:: b32hexdecode(s, casefold=False) + +.. function:: b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b'', canonical=False) Similar to :func:`b32decode` but uses the Extended Hex Alphabet, as defined in :rfc:`4648`. @@ -164,14 +242,24 @@ POST request. .. versionadded:: 3.10 + .. versionchanged:: 3.15 + Added the *canonical*, *ignorechars*, and *padded* parameters. -.. function:: b16encode(s) + +.. function:: b16encode(s, *, wrapcol=0) Encode the :term:`bytes-like object` *s* using Base16 and return the encoded :class:`bytes`. + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not add any newlines. -.. function:: b16decode(s, casefold=False) + .. versionchanged:: 3.15 + Added the *wrapcol* parameter. + + +.. function:: b16decode(s, casefold=False, *, ignorechars=b'') Decode the Base16 encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. @@ -180,10 +268,17 @@ POST request. lowercase alphabet is acceptable as input. For security purposes, the default is ``False``. + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + A :exc:`binascii.Error` is raised if *s* is incorrectly padded or if there are non-alphabet characters present in the input. + .. versionchanged:: 3.15 + Added the *ignorechars* parameter. + + .. _base64-base-85: Base85 Encodings @@ -214,12 +309,13 @@ Refer to the documentation of the individual functions for more information. instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This feature is not supported by the "standard" Ascii85 encoding. - *wrapcol* controls whether the output should have newline (``b'\n'``) - characters added to it. If this is non-zero, each output line will be - at most this many characters long, excluding the trailing newline. + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. - *pad* controls whether the input is padded to a multiple of 4 - before encoding. Note that the ``btoa`` implementation always pads. + If *pad* is true, the input is padded with ``b'\0'`` so its length is a + multiple of 4 bytes before encoding. + Note that the ``btoa`` implementation always pads. *adobe* controls whether the encoded byte sequence is framed with ``<~`` and ``~>``, which is used by the Adobe implementation. @@ -227,7 +323,7 @@ Refer to the documentation of the individual functions for more information. .. versionadded:: 3.4 -.. function:: a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v') +.. function:: a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v', canonical=False) Decode the Ascii85 encoded :term:`bytes-like object` or ASCII string *b* and return the decoded :class:`bytes`. @@ -239,15 +335,23 @@ Refer to the documentation of the individual functions for more information. *adobe* controls whether the input sequence is in Adobe Ascii85 format (i.e. is framed with <~ and ~>). - *ignorechars* should be a :term:`bytes-like object` or ASCII string - containing characters to ignore - from the input. This should only contain whitespace characters, and by + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + This should only contain whitespace characters, and by default contains all whitespace characters in ASCII. + If *canonical* is true, non-canonical encodings are rejected. + See :func:`binascii.a2b_ascii85` for details. + .. versionadded:: 3.4 + .. versionchanged:: next + Added the *canonical* parameter. + Single-character final groups are now always rejected as encoding + violations. -.. function:: b85encode(b, pad=False) + +.. function:: b85encode(b, pad=False, *, wrapcol=0) Encode the :term:`bytes-like object` *b* using base85 (as used in e.g. git-style binary diffs) and return the encoded :class:`bytes`. @@ -255,35 +359,77 @@ Refer to the documentation of the individual functions for more information. If *pad* is true, the input is padded with ``b'\0'`` so its length is a multiple of 4 bytes before encoding. + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not add any newlines. + .. versionadded:: 3.4 + .. versionchanged:: 3.15 + Added the *wrapcol* parameter. -.. function:: b85decode(b) + +.. function:: b85decode(b, *, ignorechars=b'', canonical=False) Decode the base85-encoded :term:`bytes-like object` or ASCII string *b* and return the decoded :class:`bytes`. Padding is implicitly removed, if necessary. + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + + If *canonical* is true, non-canonical encodings are rejected. + See :func:`binascii.a2b_base85` for details. + .. versionadded:: 3.4 + .. versionchanged:: 3.15 + Added the *canonical* and *ignorechars* parameters. + Single-character final groups are now always rejected as encoding + violations. -.. function:: z85encode(s) + +.. function:: z85encode(s, pad=False, *, wrapcol=0) Encode the :term:`bytes-like object` *s* using Z85 (as used in ZeroMQ) and return the encoded :class:`bytes`. See `Z85 specification `_ for more information. + If *pad* is true, the input is padded with ``b'\0'`` so its length is a + multiple of 4 bytes before encoding. + + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not add any newlines. + .. versionadded:: 3.13 + .. versionchanged:: 3.15 + The *pad* parameter was added. -.. function:: z85decode(s) + .. versionchanged:: 3.15 + Added the *wrapcol* parameter. + + +.. function:: z85decode(s, *, ignorechars=b'', canonical=False) Decode the Z85-encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. See `Z85 specification `_ for more information. + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + + If *canonical* is true, non-canonical encodings are rejected. + See :func:`binascii.a2b_base85` for details. + .. versionadded:: 3.13 + .. versionchanged:: 3.15 + Added the *canonical* and *ignorechars* parameters. + Single-character final groups are now always rejected as encoding + violations. + .. _base64-legacy: diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index c7a3e0c596b..c8b48901901 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -8,7 +8,7 @@ -------------- -The :mod:`bdb` module handles basic debugger functions, like setting breakpoints +The :mod:`!bdb` module handles basic debugger functions, like setting breakpoints or managing execution via the debugger. The following exception is defined: @@ -18,7 +18,7 @@ The following exception is defined: Exception raised by the :class:`Bdb` class for quitting the debugger. -The :mod:`bdb` module also defines two classes: +The :mod:`!bdb` module also defines two classes: .. class:: Breakpoint(self, file, line, temporary=False, cond=None, funcname=None) @@ -236,7 +236,7 @@ The :mod:`bdb` module also defines two classes: Normally derived classes don't override the following methods, but they may if they want to redefine the definition of stopping and breakpoints. - .. method:: is_skipped_line(module_name) + .. method:: is_skipped_module(module_name) Return ``True`` if *module_name* matches any skip pattern. diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 1bab785684b..8b4ba6ae9fb 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -10,10 +10,10 @@ -------------- -The :mod:`binascii` module contains a number of methods to convert between +The :mod:`!binascii` module contains a number of methods to convert between binary and various ASCII-encoded binary representations. Normally, you will not use these functions directly but use wrapper modules like -:mod:`base64` instead. The :mod:`binascii` module contains +:mod:`base64` instead. The :mod:`!binascii` module contains low-level functions written in C for greater speed that are used by the higher-level modules. @@ -28,7 +28,7 @@ higher-level modules. ASCII-only unicode strings are now accepted by the ``a2b_*`` functions. -The :mod:`binascii` module defines the following functions: +The :mod:`!binascii` module defines the following functions: .. function:: a2b_uu(string) @@ -48,34 +48,231 @@ The :mod:`binascii` module defines the following functions: Added the *backtick* parameter. -.. function:: a2b_base64(string, /, *, strict_mode=False) +.. function:: a2b_base64(string, /, *, padded=True, alphabet=BASE64_ALPHABET, strict_mode=False, canonical=False) + a2b_base64(string, /, *, ignorechars, padded=True, alphabet=BASE64_ALPHABET, strict_mode=True, canonical=False) Convert a block of base64 data back to binary and return the binary data. More than one line may be passed at a time. + Optional *alphabet* must be a :class:`bytes` object of length 64 which + specifies an alternative alphabet. + + If *padded* is true, the last group of 4 base 64 alphabet characters must + be padded with the '=' character. + If *padded* is false, padding is neither required nor recognized: + the '=' character is not treated as padding but as a non-alphabet + character, which means it is silently discarded when *strict_mode* is false, + or causes an :exc:`~binascii.Error` when *strict_mode* is true unless + b'=' is included in *ignorechars*. + + If *ignorechars* is specified, it should be a :term:`bytes-like object` + containing characters to ignore from the input when *strict_mode* is true. + If *ignorechars* contains the pad character ``'='``, the pad characters + presented before the end of the encoded data and the excess pad characters + will be ignored. + The default value of *strict_mode* is ``True`` if *ignorechars* is specified, + ``False`` otherwise. + If *strict_mode* is true, only valid base64 data will be converted. Invalid base64 data will raise :exc:`binascii.Error`. Valid base64: - * Conforms to :rfc:`3548`. + * Conforms to :rfc:`4648`. * Contains only characters from the base64 alphabet. * Contains no excess data after padding (including excess padding, newlines, etc.). * Does not start with a padding. + If *canonical* is true, non-zero padding bits in the last group are rejected + with :exc:`binascii.Error`, enforcing canonical encoding as defined in + :rfc:`4648` section 3.5. This check is independent of *strict_mode*. + .. versionchanged:: 3.11 Added the *strict_mode* parameter. + .. versionchanged:: 3.15 + Added the *alphabet*, *canonical*, *ignorechars*, and *padded* parameters. -.. function:: b2a_base64(data, *, newline=True) - Convert binary data to a line of ASCII characters in base64 coding. The return - value is the converted line, including a newline char if *newline* is - true. The output of this function conforms to :rfc:`3548`. +.. function:: b2a_base64(data, *, padded=True, alphabet=BASE64_ALPHABET, wrapcol=0, newline=True) + + Convert binary data to a line(s) of ASCII characters in base64 coding, + as specified in :rfc:`4648`. + + If *padded* is true (default), pad the encoded data with the '=' + character to a size multiple of 4. + If *padded* is false, do not add the pad characters. + + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. + + If *newline* is true (default), a newline character will be added + at the end of the output. .. versionchanged:: 3.6 Added the *newline* parameter. + .. versionchanged:: 3.15 + Added the *alphabet*, *padded* and *wrapcol* parameters. + + +.. function:: a2b_ascii85(string, /, *, foldspaces=False, adobe=False, ignorechars=b'', canonical=False) + + Convert Ascii85 data back to binary and return the binary data. + + Valid Ascii85 data contains characters from the Ascii85 alphabet in groups + of five (except for the final group, which may have from two to five + characters). Each group encodes 32 bits of binary data in the range from + ``0`` to ``2 ** 32 - 1``, inclusive. The special character ``z`` is + accepted as a short form of the group ``!!!!!``, which encodes four + consecutive null bytes. A single-character final group is always rejected + as an encoding violation. + + *foldspaces* is a flag that specifies whether the 'y' short sequence + should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). + This feature is not supported by the "standard" Ascii85 encoding. + + *adobe* controls whether the input sequence is in Adobe Ascii85 format + (i.e. is framed with <~ and ~>). + + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + This should only contain whitespace characters. + + If *canonical* is true, non-canonical encodings are rejected with + :exc:`binascii.Error`. Here "canonical" means the encoding that + :func:`b2a_ascii85` would produce: the ``z`` abbreviation must be used + for all-zero groups (rather than ``!!!!!``), and partial final groups + must use the same padding digits as the encoder. + + Invalid Ascii85 data will raise :exc:`binascii.Error`. + + .. versionadded:: 3.15 + + +.. function:: b2a_ascii85(data, /, *, foldspaces=False, wrapcol=0, pad=False, adobe=False) + + Convert binary data to a formatted sequence of ASCII characters in Ascii85 + coding. The return value is the converted data. + + *foldspaces* is an optional flag that uses the special short sequence 'y' + instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This + feature is not supported by the "standard" Ascii85 encoding. + + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. + + If *pad* is true, the input is padded with ``b'\0'`` so its length is a + multiple of 4 bytes before encoding. + Note that the ``btoa`` implementation always pads. + + *adobe* controls whether the encoded byte sequence is framed with ``<~`` + and ``~>``, which is used by the Adobe implementation. + + .. versionadded:: 3.15 + + +.. function:: a2b_base85(string, /, *, alphabet=BASE85_ALPHABET, ignorechars=b'', canonical=False) + + Convert Base85 data back to binary and return the binary data. + More than one line may be passed at a time. + + Valid Base85 data contains characters from the Base85 alphabet in groups + of five (except for the final group, which may have from two to five + characters). Each group encodes 32 bits of binary data in the range from + ``0`` to ``2 ** 32 - 1``, inclusive. A single-character final group is + always rejected as an encoding violation. + + Optional *alphabet* must be a :class:`bytes` object of length 85 which + specifies an alternative alphabet. + + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + + If *canonical* is true, non-canonical encodings are rejected with + :exc:`binascii.Error`. Here "canonical" means the encoding that + :func:`b2a_base85` would produce: partial final groups must use the + same padding digits as the encoder. + + Invalid Base85 data will raise :exc:`binascii.Error`. + + .. versionadded:: 3.15 + + +.. function:: b2a_base85(data, /, *, alphabet=BASE85_ALPHABET, wrapcol=0, pad=False) + + Convert binary data to a line of ASCII characters in Base85 coding. + The return value is the converted line. + + Optional *alphabet* must be a :term:`bytes-like object` of length 85 which + specifies an alternative alphabet. + + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. + + If *pad* is true, the input is padded with ``b'\0'`` so its length is a + multiple of 4 bytes before encoding. + + .. versionadded:: 3.15 + + +.. function:: a2b_base32(string, /, *, padded=True, alphabet=BASE32_ALPHABET, ignorechars=b'', canonical=False) + + Convert base32 data back to binary and return the binary data. + + Valid base32 data contains characters from the base32 alphabet specified + in :rfc:`4648` in groups of eight (if necessary, the final group is padded + to eight characters with ``=``). Each group encodes 40 bits of binary data + in the range from ``0`` to ``2 ** 40 - 1``, inclusive. + + .. note:: + This function does not map lowercase characters (which are invalid in + standard base32) to their uppercase counterparts, nor does it + contextually map ``0`` to ``O`` and ``1`` to ``I``/``L`` as :rfc:`4648` + allows. + + Optional *alphabet* must be a :class:`bytes` object of length 32 which + specifies an alternative alphabet. + + If *padded* is true, the last group of 8 base 32 alphabet characters must + be padded with the '=' character. + If *padded* is false, the '=' character is treated as other non-alphabet + characters (depending on the value of *ignorechars*). + + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + If *ignorechars* contains the pad character ``'='``, the pad characters + presented before the end of the encoded data and the excess pad characters + will be ignored. + + If *canonical* is true, non-zero padding bits in the last group are rejected + with :exc:`binascii.Error`, enforcing canonical encoding as defined in + :rfc:`4648` section 3.5. + + Invalid base32 data will raise :exc:`binascii.Error`. + + .. versionadded:: 3.15 + +.. function:: b2a_base32(data, /, *, padded=True, alphabet=BASE32_ALPHABET, wrapcol=0) + + Convert binary data to a line of ASCII characters in base32 coding, + as specified in :rfc:`4648`. The return value is the converted line. + + Optional *alphabet* must be a :term:`bytes-like object` of length 32 which + specifies an alternative alphabet. + + If *padded* is true (default), pad the encoded data with the '=' + character to a size multiple of 8. + If *padded* is false, do not add the pad characters. + + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. + + .. versionadded:: 3.15 .. function:: a2b_qp(data, header=False) @@ -150,18 +347,25 @@ The :mod:`binascii` module defines the following functions: .. versionchanged:: 3.8 The *sep* and *bytes_per_sep* parameters were added. -.. function:: a2b_hex(hexstr) - unhexlify(hexstr) +.. function:: a2b_hex(hexstr, *, ignorechars=b'') + unhexlify(hexstr, *, ignorechars=b'') Return the binary data represented by the hexadecimal string *hexstr*. This function is the inverse of :func:`b2a_hex`. *hexstr* must contain an even number of hexadecimal digits (which can be upper or lower case), otherwise an :exc:`Error` exception is raised. + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + Similar functionality (accepting only text string arguments, but more liberal towards whitespace) is also accessible using the :meth:`bytes.fromhex` class method. + .. versionchanged:: 3.15 + Added the *ignorechars* parameter. + + .. exception:: Error Exception raised on errors. These are usually programming errors. @@ -173,6 +377,69 @@ The :mod:`binascii` module defines the following functions: but may be handled by reading a little more data and trying again. +.. data:: BASE64_ALPHABET + + The Base 64 alphabet according to :rfc:`4648`. + + .. versionadded:: 3.15 + +.. data:: URLSAFE_BASE64_ALPHABET + + The "URL and filename safe" Base 64 alphabet according to :rfc:`4648`. + + .. versionadded:: 3.15 + +.. data:: UU_ALPHABET + + The uuencoding alphabet. + + .. versionadded:: 3.15 + +.. data:: CRYPT_ALPHABET + + The Base 64 alphabet used in the :manpage:`crypt(3)` routine and in the GEDCOM format. + + .. versionadded:: 3.15 + +.. data:: BINHEX_ALPHABET + + The Base 64 alphabet used in BinHex 4 (HQX) within the classic Mac OS. + + .. versionadded:: 3.15 + +.. data:: BASE85_ALPHABET + + The Base85 alphabet. + + .. versionadded:: 3.15 + +.. data:: ASCII85_ALPHABET + + The Ascii85 alphabet. + + .. versionadded:: 3.15 + +.. data:: Z85_ALPHABET + + The `Z85 `_ alphabet. + + .. versionadded:: 3.15 + +.. data:: BASE32_ALPHABET + + The Base 32 alphabet according to :rfc:`4648`. + + .. versionadded:: 3.15 + +.. data:: BASE32HEX_ALPHABET + + The "Extended Hex" Base 32 alphabet according to :rfc:`4648`. + Data encoded with this alphabet maintains its sort order during bitwise + comparisons. + + .. versionadded:: 3.15 + + .. seealso:: Module :mod:`base64` diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index d5ec4212c1f..2c29a5ec992 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -3,9 +3,6 @@ .. module:: bisect :synopsis: Array bisection algorithms for binary searching. -.. sectionauthor:: Fred L. Drake, Jr. -.. sectionauthor:: Raymond Hettinger -.. example based on the PyModules FAQ entry by Aaron Watters **Source code:** :source:`Lib/bisect.py` @@ -16,7 +13,7 @@ having to sort the list after each insertion. For long lists of items with expensive comparison operations, this can be an improvement over linear searches or frequent resorting. -The module is called :mod:`bisect` because it uses a basic bisection +The module is called :mod:`!bisect` because it uses a basic bisection algorithm to do its work. Unlike other bisection tools that search for a specific value, the functions in this module are designed to locate an insertion point. Accordingly, the functions never call an :meth:`~object.__eq__` @@ -27,9 +24,9 @@ point between values in an array. .. note:: The functions in this module are not thread-safe. If multiple threads - concurrently use :mod:`bisect` functions on the same sequence, this + concurrently use :mod:`!bisect` functions on the same sequence, this may result in undefined behaviour. Likewise, if the provided sequence - is mutated by a different thread while a :mod:`bisect` function + is mutated by a different thread while a :mod:`!bisect` function is operating on it, the result is undefined. For example, using :py:func:`~bisect.insort_left` on the same list from multiple threads may result in the list becoming unsorted. @@ -203,9 +200,9 @@ example uses :py:func:`~bisect.bisect` to look up a letter grade for an exam sco based on a set of ordered numeric breakpoints: 90 and up is an 'A', 80 to 89 is a 'B', and so on:: - >>> def grade(score, breakpoints=[60, 70, 80, 90], grades='FDCBA'): - ... i = bisect(breakpoints, score) - ... return grades[i] + >>> def grade(score) + ... i = bisect([60, 70, 80, 90], score) + ... return "FDCBA"[i] ... >>> [grade(score) for score in [33, 99, 77, 70, 89, 90, 100]] ['F', 'A', 'C', 'C', 'B', 'A', 'A'] diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index ebe2e43feba..6c20e9c94a3 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -4,11 +4,6 @@ .. module:: bz2 :synopsis: Interfaces for bzip2 compression and decompression. -.. moduleauthor:: Gustavo Niemeyer -.. moduleauthor:: Nadeem Vawda -.. sectionauthor:: Gustavo Niemeyer -.. sectionauthor:: Nadeem Vawda - **Source code:** :source:`Lib/bz2.py` -------------- @@ -16,7 +11,7 @@ This module provides a comprehensive interface for compressing and decompressing data using the bzip2 compression algorithm. -The :mod:`bz2` module contains: +The :mod:`!bz2` module contains: * The :func:`.open` function and :class:`BZ2File` class for reading and writing compressed files. @@ -25,6 +20,8 @@ The :mod:`bz2` module contains: * The :func:`compress` and :func:`decompress` functions for one-shot (de)compression. +.. include:: ../includes/optional-module.rst + (De)compression of files ------------------------ @@ -315,7 +312,7 @@ One-shot (de)compression Examples of usage ----------------- -Below are some examples of typical usage of the :mod:`bz2` module. +Below are some examples of typical usage of the :mod:`!bz2` module. Using :func:`compress` and :func:`decompress` to demonstrate round-trip compression: diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index fd397547a04..1c8f25e96dc 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -5,8 +5,6 @@ :synopsis: Functions for working with calendars, including some emulation of the Unix cal program. -.. sectionauthor:: Drew Csillag - **Source code:** :source:`Lib/calendar.py` -------------- @@ -56,13 +54,13 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is .. method:: setfirstweekday(firstweekday) - Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6) + Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6). Identical to setting the :attr:`~Calendar.firstweekday` property. .. method:: iterweekdays() - Return an iterator for the week day numbers that will be used for one + Return an iterator for the weekday numbers that will be used for one week. The first value from the iterator will be the same as the value of the :attr:`~Calendar.firstweekday` property. @@ -88,7 +86,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is Return an iterator for the month *month* in the year *year* similar to :meth:`itermonthdates`, but not restricted by the :class:`datetime.date` range. Days returned will be tuples consisting of a day of the month - number and a week day number. + number and a weekday number. .. method:: itermonthdays3(year, month) @@ -158,6 +156,11 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is :class:`TextCalendar` instances have the following methods: + .. method:: prweek(theweek, width) + + Print a week's calendar as returned by :meth:`formatweek` and without a + final newline. + .. method:: formatday(theday, weekday, width) @@ -405,7 +408,7 @@ For simple text calendars this module provides the following functions. .. function:: monthrange(year, month) - Returns weekday of first day of the month and number of days in month, for the + Returns weekday of first day of the month and number of days in month, for the specified *year* and *month*. @@ -443,11 +446,11 @@ For simple text calendars this module provides the following functions. An unrelated but handy function that takes a time tuple such as returned by the :func:`~time.gmtime` function in the :mod:`time` module, and returns the corresponding Unix timestamp value, assuming an epoch of 1970, and the POSIX - encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each others' + encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each other's inverse. -The :mod:`calendar` module exports the following data attributes: +The :mod:`!calendar` module exports the following data attributes: .. data:: day_name @@ -533,7 +536,7 @@ The :mod:`calendar` module exports the following data attributes: in the standalone form if the locale provides one. Else it is equivalent to :data:`month_name`. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: standalone_month_abbr @@ -542,7 +545,7 @@ The :mod:`calendar` module exports the following data attributes: locale in the standalone form if the locale provides one. Else it is equivalent to :data:`month_abbr`. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: JANUARY @@ -573,13 +576,18 @@ The :mod:`calendar` module exports the following data attributes: .. versionadded:: 3.12 -The :mod:`calendar` module defines the following exceptions: +The :mod:`!calendar` module defines the following exceptions: .. exception:: IllegalMonthError(month) - A subclass of :exc:`ValueError`, + A subclass of :exc:`ValueError` and :exc:`IndexError`, raised when the given month number is outside of the range 1-12 (inclusive). + .. versionchanged:: 3.12 + :exc:`IllegalMonthError` is now also a subclass of + :exc:`ValueError`. New code should avoid catching + :exc:`IndexError`. + .. attribute:: month The invalid month number. @@ -612,7 +620,7 @@ Command-line usage .. versionadded:: 2.5 -The :mod:`calendar` module can be executed as a script from the command line +The :mod:`!calendar` module can be executed as a script from the command line to interactively print a calendar. .. code-block:: shell @@ -710,8 +718,7 @@ The following options are accepted: .. option:: month The month of the specified :option:`year` to print the calendar for. - Must be a number between 1 and 12, - and may only be used in text mode. + Must be a number between 1 and 12. Defaults to printing a calendar for the full year. diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index 26518a0458f..f602003e49b 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -124,7 +124,7 @@ rectangular coordinates to polar coordinates and back. The modulus (absolute value) of a complex number *z* can be computed using the built-in :func:`abs` function. There is no - separate :mod:`cmath` module function for this operation. + separate :mod:`!cmath` module function for this operation. .. function:: polar(z) @@ -338,7 +338,7 @@ Constants .. data:: nan A floating-point "not a number" (NaN) value. Equivalent to - ``float('nan')``. + ``float('nan')``. See also :data:`math.nan`. .. versionadded:: 3.6 @@ -357,7 +357,7 @@ Note that the selection of functions is similar, but not identical, to that in module :mod:`math`. The reason for having two modules is that some users aren't interested in complex numbers, and perhaps don't even know what they are. They would rather have ``math.sqrt(-1)`` raise an exception than return a complex -number. Also note that the functions defined in :mod:`cmath` always return a +number. Also note that the functions defined in :mod:`!cmath` always return a complex number, even if the answer can be expressed as a real number (in which case the complex number has an imaginary part of zero). diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst index 66544f82f6f..c988fcebd68 100644 --- a/Doc/library/cmd.rst +++ b/Doc/library/cmd.rst @@ -4,8 +4,6 @@ .. module:: cmd :synopsis: Build line-oriented command interpreters. -.. sectionauthor:: Eric S. Raymond - **Source code:** :source:`Lib/cmd.py` -------------- @@ -243,9 +241,7 @@ Instances of :class:`Cmd` subclasses have some public instance variables: Cmd Example ----------- -.. sectionauthor:: Raymond Hettinger - -The :mod:`cmd` module is mainly useful for building custom shells that let a +The :mod:`!cmd` module is mainly useful for building custom shells that let a user work with a program interactively. This section presents a simple example of how to build a shell around a few of diff --git a/Doc/library/cmdline.rst b/Doc/library/cmdline.rst index 16c67ddbf7c..6418706269f 100644 --- a/Doc/library/cmdline.rst +++ b/Doc/library/cmdline.rst @@ -12,27 +12,28 @@ The following modules have a command-line interface. * :ref:`calendar ` * :mod:`code` * :ref:`compileall ` -* :mod:`cProfile`: see :ref:`profile ` +* ``cProfile``: see :ref:`profiling.tracing ` * :ref:`dis ` * :ref:`doctest ` * :mod:`!encodings.rot_13` -* :mod:`ensurepip` +* :ref:`ensurepip ` * :mod:`filecmp` * :mod:`fileinput` * :mod:`ftplib` * :ref:`gzip ` * :ref:`http.server ` -* :mod:`!idlelib` +* :ref:`idlelib ` * :ref:`inspect ` * :ref:`json ` * :ref:`mimetypes ` -* :mod:`pdb` +* :ref:`pdb ` * :ref:`pickle ` * :ref:`pickletools ` * :ref:`platform ` * :mod:`poplib` -* :ref:`profile ` -* :mod:`pstats` +* :ref:`profiling.sampling ` +* :ref:`profiling.tracing ` +* :ref:`pstats ` * :ref:`py_compile ` * :mod:`pyclbr` * :mod:`pydoc` @@ -52,8 +53,8 @@ The following modules have a command-line interface. * :mod:`turtledemo` * :ref:`unittest ` * :ref:`uuid ` -* :mod:`venv` -* :mod:`webbrowser` +* :ref:`venv ` +* :ref:`webbrowser ` * :ref:`zipapp ` * :ref:`zipfile ` diff --git a/Doc/library/code.rst b/Doc/library/code.rst index 52587c4dd8f..59c016d2150 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -29,7 +29,7 @@ build applications which provide an interactive interpreter prompt. module. -.. class:: InteractiveConsole(locals=None, filename="", local_exit=False) +.. class:: InteractiveConsole(locals=None, filename="", *, local_exit=False) Closely emulate the behavior of the interactive Python interpreter. This class builds on :class:`InteractiveInterpreter` and adds prompting using the familiar diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 90a695ef937..9259ab10d58 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -4,10 +4,6 @@ .. module:: codecs :synopsis: Encode and decode data and streams. -.. moduleauthor:: Marc-André Lemburg -.. sectionauthor:: Marc-André Lemburg -.. sectionauthor:: Martin v. Löwis - **Source code:** :source:`Lib/codecs.py` .. index:: @@ -78,7 +74,7 @@ The full details for each codec can also be looked up directly: .. versionchanged:: 3.9 Any characters except ASCII letters and digits and a dot are converted to underscore. - .. versionchanged:: next + .. versionchanged:: 3.15 No characters are converted to underscore anymore. Spaces are converted to hyphens. @@ -317,7 +313,7 @@ and writing to platform dependent files: Codec Base Classes ------------------ -The :mod:`codecs` module defines a set of base classes which define the +The :mod:`!codecs` module defines a set of base classes which define the interfaces for working with codec objects, and can also be used as the basis for custom codec implementations. @@ -989,17 +985,22 @@ defined in Unicode. A simple and straightforward way that can store each Unicode code point, is to store each code point as four consecutive bytes. There are two possibilities: store the bytes in big endian or in little endian order. These two encodings are called ``UTF-32-BE`` and ``UTF-32-LE`` respectively. Their -disadvantage is that if e.g. you use ``UTF-32-BE`` on a little endian machine you -will always have to swap bytes on encoding and decoding. ``UTF-32`` avoids this -problem: bytes will always be in natural endianness. When these bytes are read -by a CPU with a different endianness, then bytes have to be swapped though. To -be able to detect the endianness of a ``UTF-16`` or ``UTF-32`` byte sequence, -there's the so called BOM ("Byte Order Mark"). This is the Unicode character -``U+FEFF``. This character can be prepended to every ``UTF-16`` or ``UTF-32`` -byte sequence. The byte swapped version of this character (``0xFFFE``) is an -illegal character that may not appear in a Unicode text. So when the -first character in a ``UTF-16`` or ``UTF-32`` byte sequence -appears to be a ``U+FFFE`` the bytes have to be swapped on decoding. +disadvantage is that if, for example, you use ``UTF-32-BE`` on a little endian +machine you will always have to swap bytes on encoding and decoding. +Python's ``UTF-16`` and ``UTF-32`` codecs avoid this problem by using the +platform's native byte order when no BOM is present. +Python follows prevailing platform +practice, so native-endian data round-trips without redundant byte swapping, +even though the Unicode Standard defaults to big-endian when the byte order is +unspecified. When these bytes are read by a CPU with a different endianness, +the bytes have to be swapped. To be able to detect the endianness of a +``UTF-16`` or ``UTF-32`` byte sequence, a BOM ("Byte Order Mark") is used. +This is the Unicode character ``U+FEFF``. This character can be prepended to every +``UTF-16`` or ``UTF-32`` byte sequence. The byte swapped version of this character +(``0xFFFE``) is an illegal character that may not appear in a Unicode text. +When the first character of a ``UTF-16`` or ``UTF-32`` byte sequence is +``U+FFFE``, the bytes have to be swapped on decoding. + Unfortunately the character ``U+FEFF`` had a second purpose as a ``ZERO WIDTH NO-BREAK SPACE``: a character that has no width and doesn't allow a word to be split. It can e.g. be used to give hints to a ligature algorithm. @@ -1083,7 +1084,7 @@ alias for the ``'utf_8'`` codec. refer to the source :source:`aliases.py ` file. On Windows, ``cpXXX`` codecs are available for all code pages. -But only codecs listed in the following table are guarantead to exist on +But only codecs listed in the following table are guaranteed to exist on other platforms. .. impl-detail:: @@ -1339,7 +1340,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | utf_8 | U8, UTF, utf8, cp65001 | all languages | +-----------------+--------------------------------+--------------------------------+ -| utf_8_sig | | all languages | +| utf_8_sig | utf8-sig | all languages | +-----------------+--------------------------------+--------------------------------+ .. versionchanged:: 3.4 @@ -1546,8 +1547,8 @@ mapping. It is not supported by :meth:`str.encode` (which only produces Restoration of the ``rot13`` alias. -:mod:`encodings` --- Encodings package --------------------------------------- +:mod:`!encodings` --- Encodings package +--------------------------------------- .. module:: encodings :synopsis: Encodings package @@ -1606,12 +1607,11 @@ This module implements the following exception: Raised when a codec is invalid or incompatible. -:mod:`encodings.idna` --- Internationalized Domain Names in Applications ------------------------------------------------------------------------- +:mod:`!encodings.idna` --- Internationalized Domain Names in Applications +------------------------------------------------------------------------- .. module:: encodings.idna :synopsis: Internationalized Domain Names implementation -.. moduleauthor:: Martin v. Löwis This module implements :rfc:`3490` (Internationalized Domain Names in Applications) and :rfc:`3492` (Nameprep: A Stringprep Profile for @@ -1649,7 +1649,7 @@ When receiving host names from the wire (such as in reverse name lookup), no automatic conversion to Unicode is performed: applications wishing to present such host names to the user should decode them to Unicode. -The module :mod:`encodings.idna` also implements the nameprep procedure, which +The module :mod:`!encodings.idna` also implements the nameprep procedure, which performs certain normalizations on host names, to achieve case-insensitivity of international domain names, and to unify similar characters. The nameprep functions can be used directly if desired. @@ -1672,8 +1672,8 @@ functions can be used directly if desired. Convert a label to Unicode, as specified in :rfc:`3490`. -:mod:`encodings.mbcs` --- Windows ANSI codepage ------------------------------------------------ +:mod:`!encodings.mbcs` --- Windows ANSI codepage +------------------------------------------------ .. module:: encodings.mbcs :synopsis: Windows ANSI codepage @@ -1690,12 +1690,11 @@ This module implements the ANSI codepage (CP_ACP). Support any error handler. -:mod:`encodings.utf_8_sig` --- UTF-8 codec with BOM signature -------------------------------------------------------------- +:mod:`!encodings.utf_8_sig` --- UTF-8 codec with BOM signature +-------------------------------------------------------------- .. module:: encodings.utf_8_sig :synopsis: UTF-8 codec with BOM signature -.. moduleauthor:: Walter Dörwald This module implements a variant of the UTF-8 codec. On encoding, a UTF-8 encoded BOM will be prepended to the UTF-8 encoded bytes. For the stateful encoder this diff --git a/Doc/library/codeop.rst b/Doc/library/codeop.rst index 16f674adb4b..622e57d2ee6 100644 --- a/Doc/library/codeop.rst +++ b/Doc/library/codeop.rst @@ -4,14 +4,11 @@ .. module:: codeop :synopsis: Compile (possibly incomplete) Python code. -.. sectionauthor:: Moshe Zadka -.. sectionauthor:: Michael Hudson - **Source code:** :source:`Lib/codeop.py` -------------- -The :mod:`codeop` module provides utilities upon which the Python +The :mod:`!codeop` module provides utilities upon which the Python read-eval-print loop can be emulated, as is done in the :mod:`code` module. As a result, you probably don't want to use the module directly; if you want to include such a loop in your program you probably want to use the :mod:`code` @@ -25,7 +22,7 @@ There are two parts to this job: #. Remembering which future statements the user has entered, so subsequent input can be compiled with these in effect. -The :mod:`codeop` module provides a way of doing each of these things, and a way +The :mod:`!codeop` module provides a way of doing each of these things, and a way of doing them both. To do just the former: diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index db9277ff09b..51853725b1b 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -4,9 +4,6 @@ .. module:: collections.abc :synopsis: Abstract base classes for containers -.. moduleauthor:: Raymond Hettinger -.. sectionauthor:: Raymond Hettinger - .. versionadded:: 3.3 Formerly, this module was part of the :mod:`collections` module. @@ -140,6 +137,9 @@ ABC Inherits from Abstract Methods Mi ``__len__``, ``insert`` +:class:`ByteString` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods + ``__len__`` + :class:`Set` :class:`Collection` ``__contains__``, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``, ``__iter__``, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``, ``__len__`` ``__sub__``, ``__rsub__``, ``__xor__``, ``__rxor__`` @@ -260,6 +260,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Sequence MutableSequence + ByteString ABCs for read-only and mutable :term:`sequences `. @@ -285,6 +286,25 @@ Collections Abstract Base Classes -- Detailed Descriptions The :meth:`~sequence.index` method gained support for the *stop* and *start* arguments. + .. deprecated-removed:: 3.12 3.17 + The :class:`ByteString` ABC has been deprecated. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the :ref:`buffer protocol ` at runtime. For use + in type annotations, either use :class:`Buffer` or a union that + explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was + an instance of :class:`!ByteString` never actually told you anything + useful about the object. Other common buffer types such as + :class:`memoryview` were also never understood as subtypes of + :class:`!ByteString` (either at runtime or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. + .. class:: Set MutableSet @@ -313,7 +333,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. note:: In CPython, generator-based coroutines (:term:`generators ` - decorated with :func:`@types.coroutine `) are + decorated with :deco:`types.coroutine`) are *awaitables*, even though they do not have an :meth:`~object.__await__` method. Using ``isinstance(gencoro, Awaitable)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. @@ -331,7 +351,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. note:: In CPython, generator-based coroutines (:term:`generators ` - decorated with :func:`@types.coroutine `) are + decorated with :deco:`types.coroutine`) are *awaitables*, even though they do not have an :meth:`~object.__await__` method. Using ``isinstance(gencoro, Coroutine)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index fdd31799bd9..e42bdc06be0 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -4,9 +4,6 @@ .. module:: collections :synopsis: Container datatypes -.. moduleauthor:: Raymond Hettinger -.. sectionauthor:: Raymond Hettinger - **Source code:** :source:`Lib/collections/__init__.py` .. testsetup:: * @@ -240,7 +237,9 @@ For example:: [('the', 1143), ('and', 966), ('to', 762), ('of', 669), ('i', 631), ('you', 554), ('a', 546), ('my', 514), ('hamlet', 471), ('in', 451)] -.. class:: Counter([iterable-or-mapping]) +.. class:: Counter(**kwargs) + Counter(iterable, /, **kwargs) + Counter(mapping, /, **kwargs) A :class:`Counter` is a :class:`dict` subclass for counting :term:`hashable` objects. It is a collection where elements are stored as dictionary keys @@ -290,7 +289,7 @@ For example:: >>> sorted(c.elements()) ['a', 'a', 'a', 'a', 'b', 'b'] - .. method:: most_common([n]) + .. method:: most_common(n=None) Return a list of the *n* most common elements and their counts from the most common to the least. If *n* is omitted or ``None``, @@ -300,7 +299,9 @@ For example:: >>> Counter('abracadabra').most_common(3) [('a', 5), ('b', 2), ('r', 2)] - .. method:: subtract([iterable-or-mapping]) + .. method:: subtract(**kwargs) + subtract(iterable, /, **kwargs) + subtract(mapping, /, **kwargs) Elements are subtracted from an *iterable* or from another *mapping* (or counter). Like :meth:`dict.update` but subtracts counts instead @@ -325,13 +326,15 @@ For example:: .. versionadded:: 3.10 The usual dictionary methods are available for :class:`Counter` objects - except for two which work differently for counters. + except for these two which work differently for counters: .. method:: fromkeys(iterable) This class method is not implemented for :class:`Counter` objects. - .. method:: update([iterable-or-mapping]) + .. method:: update(**kwargs) + update(iterable, /, **kwargs) + update(mapping, /, **kwargs) Elements are counted from an *iterable* or added-in from another *mapping* (or counter). Like :meth:`dict.update` but adds counts @@ -367,9 +370,11 @@ Several mathematical operations are provided for combining :class:`Counter` objects to produce multisets (counters that have counts greater than zero). Addition and subtraction combine counters by adding or subtracting the counts of corresponding elements. Intersection and union return the minimum and -maximum of corresponding counts. Equality and inclusion compare -corresponding counts. Each operation can accept inputs with signed -counts, but the output will exclude results with counts of zero or less. +maximum of corresponding counts. Symmetric difference returns the difference +between the maximum and minimum of the corresponding counts. Equality and +inclusion compare corresponding counts. Each operation can accept inputs +with signed counts, but the output will exclude results with counts of zero +or below. .. doctest:: @@ -383,6 +388,8 @@ counts, but the output will exclude results with counts of zero or less. Counter({'a': 1, 'b': 1}) >>> c | d # union: max(c[x], d[x]) Counter({'a': 3, 'b': 2}) + >>> c ^ d # max(c[x], d[x]) - min(c[x], d[x]) + Counter({'a': 2, 'b': 1}) >>> c == d # equality: c[x] == d[x] False >>> c <= d # inclusion: c[x] <= d[x] @@ -400,6 +407,9 @@ or subtracting from an empty counter. .. versionadded:: 3.3 Added support for unary plus, unary minus, and in-place multiset operations. +.. versionadded:: 3.15 + Added support for the symmetric difference multiset operation, ``c ^ d``. + .. note:: Counters were primarily designed to work with positive integers to represent @@ -477,14 +487,14 @@ or subtracting from an empty counter. Deque objects support the following methods: - .. method:: append(x) + .. method:: append(item, /) - Add *x* to the right side of the deque. + Add *item* to the right side of the deque. - .. method:: appendleft(x) + .. method:: appendleft(item, /) - Add *x* to the left side of the deque. + Add *item* to the left side of the deque. .. method:: clear() @@ -499,38 +509,38 @@ or subtracting from an empty counter. .. versionadded:: 3.5 - .. method:: count(x) + .. method:: count(value, /) - Count the number of deque elements equal to *x*. + Count the number of deque elements equal to *value*. .. versionadded:: 3.2 - .. method:: extend(iterable) + .. method:: extend(iterable, /) Extend the right side of the deque by appending elements from the iterable argument. - .. method:: extendleft(iterable) + .. method:: extendleft(iterable, /) Extend the left side of the deque by appending elements from *iterable*. Note, the series of left appends results in reversing the order of elements in the iterable argument. - .. method:: index(x[, start[, stop]]) + .. method:: index(value[, start[, stop]]) - Return the position of *x* in the deque (at or after index *start* + Return the position of *value* in the deque (at or after index *start* and before index *stop*). Returns the first match or raises :exc:`ValueError` if not found. .. versionadded:: 3.5 - .. method:: insert(i, x) + .. method:: insert(index, value, /) - Insert *x* into the deque at position *i*. + Insert *value* into the deque at position *index*. If the insertion would cause a bounded deque to grow beyond *maxlen*, an :exc:`IndexError` is raised. @@ -550,7 +560,7 @@ or subtracting from an empty counter. elements are present, raises an :exc:`IndexError`. - .. method:: remove(value) + .. method:: remove(value, /) Remove the first occurrence of *value*. If not found, raises a :exc:`ValueError`. @@ -563,7 +573,7 @@ or subtracting from an empty counter. .. versionadded:: 3.2 - .. method:: rotate(n=1) + .. method:: rotate(n=1, /) Rotate the deque *n* steps to the right. If *n* is negative, rotate to the left. @@ -715,7 +725,9 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, :class:`defaultdict` objects ---------------------------- -.. class:: defaultdict(default_factory=None, /, [...]) +.. class:: defaultdict(default_factory=None, /, **kwargs) + defaultdict(default_factory, mapping, /, **kwargs) + defaultdict(default_factory, iterable, /, **kwargs) Return a new dictionary-like object. :class:`defaultdict` is a subclass of the built-in :class:`dict` class. It overrides one method and adds one writable @@ -731,7 +743,7 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, :class:`defaultdict` objects support the following method in addition to the standard :class:`dict` operations: - .. method:: __missing__(key) + .. method:: __missing__(key, /) If the :attr:`default_factory` attribute is ``None``, this raises a :exc:`KeyError` exception with the *key* as argument. @@ -937,7 +949,7 @@ In addition to the methods inherited from tuples, named tuples support three additional methods and two attributes. To prevent conflicts with field names, the method and attribute names start with an underscore. -.. classmethod:: somenamedtuple._make(iterable) +.. classmethod:: somenamedtuple._make(iterable, /) Class method that makes a new instance from an existing sequence or iterable. @@ -1134,7 +1146,9 @@ Some differences from :class:`dict` still remain: * Until Python 3.8, :class:`dict` lacked a :meth:`~object.__reversed__` method. -.. class:: OrderedDict([items]) +.. class:: OrderedDict(**kwargs) + OrderedDict(mapping, /, **kwargs) + OrderedDict(iterable, /, **kwargs) Return an instance of a :class:`dict` subclass that has methods specialized for rearranging dictionary order. @@ -1202,7 +1216,7 @@ If a new entry overwrites an existing entry, the original insertion position is changed and moved to the end:: class LastUpdatedOrderedDict(OrderedDict): - 'Store items in the order the keys were last added' + 'Store items in the order that the keys were last updated.' def __setitem__(self, key, value): super().__setitem__(key, value) @@ -1315,23 +1329,31 @@ subclass directly from :class:`dict`; however, this class can be easier to work with because the underlying dictionary is accessible as an attribute. -.. class:: UserDict([initialdata]) +.. class:: UserDict(**kwargs) + UserDict(mapping, /, **kwargs) + UserDict(iterable, /, **kwargs) Class that simulates a dictionary. The instance's contents are kept in a regular dictionary, which is accessible via the :attr:`data` attribute of - :class:`UserDict` instances. If *initialdata* is provided, :attr:`data` is - initialized with its contents; note that a reference to *initialdata* will not - be kept, allowing it to be used for other purposes. + :class:`!UserDict` instances. If arguments are provided, they are used to + initialize :attr:`data`, like a regular dictionary. In addition to supporting the methods and operations of mappings, - :class:`UserDict` instances provide the following attribute: + :class:`!UserDict` instances provide the following attribute: .. attribute:: data A real dictionary used to store the contents of the :class:`UserDict` class. + :class:`!UserDict` instances also override the following method: + .. method:: popitem + + Remove and return a ``(key, value)`` pair from the wrapped dictionary. Pairs are + returned in the same order as ``data.popitem()``. (For the default + :meth:`dict.popitem`, this order is :abbr:`LIFO (last-in, first-out)`.) If the + dictionary is empty, raises a :exc:`KeyError`. :class:`UserList` objects ------------------------- diff --git a/Doc/library/colorsys.rst b/Doc/library/colorsys.rst index ffebf4e40dd..dffc16ae8b7 100644 --- a/Doc/library/colorsys.rst +++ b/Doc/library/colorsys.rst @@ -4,13 +4,11 @@ .. module:: colorsys :synopsis: Conversion functions between RGB and other color systems. -.. sectionauthor:: David Ascher - **Source code:** :source:`Lib/colorsys.py` -------------- -The :mod:`colorsys` module defines bidirectional conversions of color values +The :mod:`!colorsys` module defines bidirectional conversions of color values between colors expressed in the RGB (Red Green Blue) color space used in computer monitors and three other coordinate systems: YIQ, HLS (Hue Lightness Saturation) and HSV (Hue Saturation Value). Coordinates in all of these color @@ -24,7 +22,7 @@ spaces, the coordinates are all between 0 and 1. https://poynton.ca/ColorFAQ.html and https://www.cambridgeincolour.com/tutorials/color-spaces.htm. -The :mod:`colorsys` module defines the following functions: +The :mod:`!colorsys` module defines the following functions: .. function:: rgb_to_yiq(r, g, b) diff --git a/Doc/library/compression.rst b/Doc/library/compression.rst index 618b4a3c2bd..98719be9992 100644 --- a/Doc/library/compression.rst +++ b/Doc/library/compression.rst @@ -1,6 +1,8 @@ The :mod:`!compression` package =============================== +.. module:: compression + .. versionadded:: 3.14 The :mod:`!compression` package contains the canonical compression modules diff --git a/Doc/library/compression.zstd.rst b/Doc/library/compression.zstd.rst index a901403621b..6d99e36e1e5 100644 --- a/Doc/library/compression.zstd.rst +++ b/Doc/library/compression.zstd.rst @@ -33,6 +33,8 @@ The :mod:`!compression.zstd` module contains: * The :class:`CompressionParameter`, :class:`DecompressionParameter`, and :class:`Strategy` classes for setting advanced (de)compression parameters. +.. include:: ../includes/optional-module.rst + Exceptions ---------- @@ -71,7 +73,7 @@ Reading and writing compressed files argument is not None, a :exc:`!TypeError` will be raised. When writing, the *options* argument can be a dictionary - providing advanced decompression parameters; see + providing advanced compression parameters; see :class:`CompressionParameter` for detailed information about supported parameters. The *level* argument is the compression level to use when writing compressed data. Only one of *level* or *options* may be non-None. @@ -115,7 +117,7 @@ Reading and writing compressed files argument is not None, a :exc:`!TypeError` will be raised. When writing, the *options* argument can be a dictionary - providing advanced decompression parameters; see + providing advanced compression parameters; see :class:`CompressionParameter` for detailed information about supported parameters. The *level* argument is the compression level to use when writing compressed data. Only one of *level* or *options* may be passed. The @@ -329,10 +331,14 @@ Compressing and decompressing data in memory If *max_length* is non-negative, the method returns at most *max_length* bytes of decompressed data. If this limit is reached and further - output can be produced, the :attr:`~.needs_input` attribute will - be set to ``False``. In this case, the next call to + output can be produced (or EOF is reached), the :attr:`~.needs_input` + attribute will be set to ``False``. In this case, the next call to :meth:`~.decompress` may provide *data* as ``b''`` to obtain - more of the output. + more of the output. The full content can thus be read like:: + + process_output(d.decompress(data, max_length)) + while not d.eof and not d.needs_input: + process_output(d.decompress(b"", max_length)) If all of the input data was decompressed and returned (either because this was less than *max_length* bytes, or because diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 6f8043e6cf7..a32c3828313 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -12,7 +12,7 @@ and :source:`Lib/concurrent/futures/interpreter.py` -------------- -The :mod:`concurrent.futures` module provides a high-level interface for +The :mod:`!concurrent.futures` module provides a high-level interface for asynchronously executing callables. The asynchronous execution can be performed with threads, using @@ -21,6 +21,11 @@ or separate processes, using :class:`ProcessPoolExecutor`. Each implements the same interface, which is defined by the abstract :class:`Executor` class. +:class:`concurrent.futures.Future` must not be confused with +:class:`asyncio.Future`, which is designed for use with :mod:`asyncio` +tasks and coroutines. See the :doc:`asyncio's Future ` +documentation for a detailed comparison of the two. + .. include:: ../includes/wasm-notavail.rst Executor Objects @@ -151,7 +156,9 @@ And:: print(f.result()) executor = ThreadPoolExecutor(max_workers=1) - executor.submit(wait_on_future) + future = executor.submit(wait_on_future) + # Note: calling future.result() would also cause a deadlock because + # the single worker thread is already waiting for wait_on_future(). .. class:: ThreadPoolExecutor(max_workers=None, thread_name_prefix='', initializer=None, initargs=()) @@ -239,6 +246,8 @@ ThreadPoolExecutor Example InterpreterPoolExecutor ----------------------- +.. versionadded:: 3.14 + The :class:`InterpreterPoolExecutor` class uses a pool of interpreters to execute calls asynchronously. It is a :class:`ThreadPoolExecutor` subclass, which means each worker is running in its own thread. @@ -306,7 +315,7 @@ the bytes over a shared :mod:`socket ` or .. note:: The executor may replace uncaught exceptions from *initializer* - with :class:`~concurrent.futures.interpreter.ExecutionFailed`. + with :class:`~concurrent.interpreters.ExecutionFailed`. Other caveats from parent :class:`ThreadPoolExecutor` apply here. @@ -318,11 +327,11 @@ likewise serializes the return value when sending it back. When a worker's current task raises an uncaught exception, the worker always tries to preserve the exception as-is. If that is successful then it also sets the ``__cause__`` to a corresponding -:class:`~concurrent.futures.interpreter.ExecutionFailed` +:class:`~concurrent.interpreters.ExecutionFailed` instance, which contains a summary of the original exception. In the uncommon case that the worker is not able to preserve the original as-is then it directly preserves the corresponding -:class:`~concurrent.futures.interpreter.ExecutionFailed` +:class:`~concurrent.interpreters.ExecutionFailed` instance instead. @@ -377,6 +386,11 @@ in a REPL or a lambda should not be expected to work. default in absence of a *mp_context* parameter. This feature is incompatible with the "fork" start method. + .. note:: + Bugs have been reported when using the *max_tasks_per_child* feature that + can result in the :class:`ProcessPoolExecutor` hanging in some + circumstances. Follow its eventual resolution in :gh:`115634`. + .. versionchanged:: 3.3 When one of the worker processes terminates abruptly, a :exc:`~concurrent.futures.process.BrokenProcessPool` error is now raised. @@ -713,15 +727,6 @@ Exception classes .. versionadded:: 3.14 -.. exception:: ExecutionFailed - - Raised from :class:`~concurrent.futures.InterpreterPoolExecutor` when - the given initializer fails or from - :meth:`~concurrent.futures.Executor.submit` when there's an uncaught - exception from the submitted task. - - .. versionadded:: 3.14 - .. currentmodule:: concurrent.futures.process .. exception:: BrokenProcessPool diff --git a/Doc/library/concurrent.interpreters.rst b/Doc/library/concurrent.interpreters.rst index 41ea6af3b22..a7b115e5f63 100644 --- a/Doc/library/concurrent.interpreters.rst +++ b/Doc/library/concurrent.interpreters.rst @@ -4,9 +4,6 @@ .. module:: concurrent.interpreters :synopsis: Multiple interpreters in the same process -.. moduleauthor:: Eric Snow -.. sectionauthor:: Eric Snow - .. versionadded:: 3.14 **Source code:** :source:`Lib/concurrent/interpreters` @@ -29,12 +26,12 @@ Actual concurrency is available separately through .. seealso:: :class:`~concurrent.futures.InterpreterPoolExecutor` - combines threads with interpreters in a familiar interface. + Combines threads with interpreters in a familiar interface. - .. XXX Add references to the upcoming HOWTO docs in the seealso block. + .. XXX Add references to the upcoming HOWTO docs in the seealso block. :ref:`isolating-extensions-howto` - how to update an extension module to support multiple interpreters + How to update an extension module to support multiple interpreters. :pep:`554` diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index bb109a9b742..4d720176fcc 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -4,13 +4,6 @@ .. module:: configparser :synopsis: Configuration file parser. -.. moduleauthor:: Ken Manheimer -.. moduleauthor:: Barry Warsaw -.. moduleauthor:: Eric S. Raymond -.. moduleauthor:: Łukasz Langa -.. sectionauthor:: Christopher G. Petrilli -.. sectionauthor:: Łukasz Langa - **Source code:** :source:`Lib/configparser.py` .. index:: @@ -31,6 +24,11 @@ can be customized by end users easily. This library does *not* interpret or write the value-type prefixes used in the Windows Registry extended version of INI syntax. +.. warning:: + Be cautious when parsing data from untrusted sources. A malicious + INI file may cause the decoder to consume considerable CPU and memory + resources. Limiting the size of data to be parsed is recommended. + .. seealso:: Module :mod:`tomllib` @@ -80,7 +78,7 @@ Let's take a very basic configuration file that looks like this: The structure of INI files is described `in the following section <#supported-ini-file-structure>`_. Essentially, the file consists of sections, each of which contains keys with values. -:mod:`configparser` classes can read and write such files. Let's start by +:mod:`!configparser` classes can read and write such files. Let's start by creating the above configuration file programmatically. .. doctest:: @@ -449,7 +447,7 @@ Mapping Protocol Access .. versionadded:: 3.2 Mapping protocol access is a generic name for functionality that enables using -custom objects as if they were dictionaries. In case of :mod:`configparser`, +custom objects as if they were dictionaries. In case of :mod:`!configparser`, the mapping interface implementation is using the ``parser['section']['option']`` notation. @@ -459,7 +457,7 @@ the original parser on demand. What's even more important is that when values are changed on a section proxy, they are actually mutated in the original parser. -:mod:`configparser` objects behave as close to actual dictionaries as possible. +:mod:`!configparser` objects behave as close to actual dictionaries as possible. The mapping interface is complete and adheres to the :class:`~collections.abc.MutableMapping` ABC. However, there are a few differences that should be taken into account: @@ -507,7 +505,7 @@ Customizing Parser Behaviour ---------------------------- There are nearly as many INI format variants as there are applications using it. -:mod:`configparser` goes a long way to provide support for the largest sensible +:mod:`!configparser` goes a long way to provide support for the largest sensible set of INI styles available. The default functionality is mainly dictated by historical background and it's very likely that you will want to customize some of the features. @@ -560,7 +558,7 @@ the :meth:`!__init__` options: * *allow_no_value*, default value: ``False`` Some configuration files are known to include settings without values, but - which otherwise conform to the syntax supported by :mod:`configparser`. The + which otherwise conform to the syntax supported by :mod:`!configparser`. The *allow_no_value* parameter to the constructor can be used to indicate that such values should be accepted: @@ -615,7 +613,7 @@ the :meth:`!__init__` options: prefixes for whole line comments. .. versionchanged:: 3.2 - In previous versions of :mod:`configparser` behaviour matched + In previous versions of :mod:`!configparser` behaviour matched ``comment_prefixes=('#',';')`` and ``inline_comment_prefixes=(';',)``. Please note that config parsers don't support escaping of comment prefixes so @@ -672,7 +670,7 @@ the :meth:`!__init__` options: parsers in new applications. .. versionchanged:: 3.2 - In previous versions of :mod:`configparser` behaviour matched + In previous versions of :mod:`!configparser` behaviour matched ``strict=False``. * *empty_lines_in_values*, default value: ``True`` @@ -842,7 +840,7 @@ be overridden by subclasses or by attribute assignment. Legacy API Examples ------------------- -Mainly because of backwards compatibility concerns, :mod:`configparser` +Mainly because of backwards compatibility concerns, :mod:`!configparser` provides also a legacy API with explicit ``get``/``set`` methods. While there are valid use cases for the methods outlined below, mapping protocol access is preferred for new projects. The legacy API is at times more advanced, @@ -1378,7 +1376,7 @@ Exceptions .. exception:: Error - Base class for all other :mod:`configparser` exceptions. + Base class for all other :mod:`!configparser` exceptions. .. exception:: NoSectionError diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 176be4ff333..77bac8dcc3a 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -21,9 +21,9 @@ Functions and classes provided: .. class:: AbstractContextManager An :term:`abstract base class` for classes that implement - :meth:`object.__enter__` and :meth:`object.__exit__`. A default - implementation for :meth:`object.__enter__` is provided which returns - ``self`` while :meth:`object.__exit__` is an abstract method which by default + :meth:`~object.__enter__` and :meth:`~object.__exit__`. A default + implementation for :meth:`~object.__enter__` is provided which returns + ``self`` while :meth:`~object.__exit__` is an abstract method which by default returns ``None``. See also the definition of :ref:`typecontextmanager`. .. versionadded:: 3.6 @@ -32,9 +32,9 @@ Functions and classes provided: .. class:: AbstractAsyncContextManager An :term:`abstract base class` for classes that implement - :meth:`object.__aenter__` and :meth:`object.__aexit__`. A default - implementation for :meth:`object.__aenter__` is provided which returns - ``self`` while :meth:`object.__aexit__` is an abstract method which by default + :meth:`~object.__aenter__` and :meth:`~object.__aexit__`. A default + implementation for :meth:`~object.__aenter__` is provided which returns + ``self`` while :meth:`~object.__aexit__` is an abstract method which by default returns ``None``. See also the definition of :ref:`async-context-managers`. @@ -228,7 +228,7 @@ Functions and classes provided: .. function:: nullcontext(enter_result=None) - Return a context manager that returns *enter_result* from ``__enter__``, but + Return a context manager that returns *enter_result* from :meth:`~object.__enter__`, but otherwise does nothing. It is intended to be used as a stand-in for an optional context manager, for example:: @@ -327,15 +327,15 @@ Functions and classes provided: .. function:: redirect_stdout(new_target) Context manager for temporarily redirecting :data:`sys.stdout` to - another file or file-like object. + another :term:`file object`. This tool adds flexibility to existing functions or classes whose output - is hardwired to stdout. + is hardwired to :data:`sys.stdout`. For example, the output of :func:`help` normally is sent to *sys.stdout*. You can capture that output in a string by redirecting the output to an :class:`io.StringIO` object. The replacement stream is returned from the - ``__enter__`` method and so is available as the target of the + :meth:`~object.__enter__` method and so is available as the target of the :keyword:`with` statement:: with redirect_stdout(io.StringIO()) as f: @@ -366,8 +366,8 @@ Functions and classes provided: .. function:: redirect_stderr(new_target) - Similar to :func:`~contextlib.redirect_stdout` but redirecting - :data:`sys.stderr` to another file or file-like object. + Similar to :func:`~contextlib.redirect_stdout` but redirecting the global + :data:`sys.stderr` to another :term:`file object`. This context manager is :ref:`reentrant `. @@ -396,7 +396,8 @@ Functions and classes provided: A base class that enables a context manager to also be used as a decorator. Context managers inheriting from ``ContextDecorator`` have to implement - ``__enter__`` and ``__exit__`` as normal. ``__exit__`` retains its optional + :meth:`~object.__enter__` and :meth:`~object.__exit__` as normal. + ``__exit__`` retains its optional exception handling even when used as a decorator. ``ContextDecorator`` is used by :func:`contextmanager`, so you get this @@ -466,12 +467,40 @@ Functions and classes provided: statements. If this is not the case, then the original construct with the explicit :keyword:`!with` statement inside the function should be used. + When the decorated callable is a generator function, coroutine function, or + asynchronous generator function, the returned wrapper is of the same kind + and keeps the context manager open for the lifetime of the iteration or + await rather than only for the call that creates the generator or coroutine + object. Wrapped generators and asynchronous generators are explicitly + closed when iteration ends, as if by :func:`closing` or :func:`aclosing`. + + .. note:: + For asynchronous generators the wrapper re-yields each value with + ``async for``; values sent with :meth:`~agen.asend` and exceptions + thrown with :meth:`~agen.athrow` are not forwarded to the wrapped + generator. + .. versionadded:: 3.2 + .. versionchanged:: next + Decorating a generator function, coroutine function, or asynchronous + generator function now keeps the context manager open across iteration + or await. Previously the context manager exited as soon as the + generator or coroutine object was created. + .. class:: AsyncContextDecorator - Similar to :class:`ContextDecorator` but only for asynchronous functions. + Similar to :class:`ContextDecorator`, but the context manager is entered + and exited with :keyword:`async with`. Decorate coroutine functions and + asynchronous generator functions with this class; the returned wrapper is + of the same kind. + + .. note:: + Synchronous functions and generators are accepted, but the wrapper is + always asynchronous, so the decorated callable must then be awaited or + iterated with ``async for``. If that change of calling convention is + not intended, use :class:`ContextDecorator` instead. Example of ``AsyncContextDecorator``:: @@ -509,6 +538,13 @@ Functions and classes provided: .. versionadded:: 3.10 + .. versionchanged:: next + Decorating an asynchronous generator function now keeps the context + manager open across iteration. Previously the context manager exited + as soon as the generator object was created. Synchronous functions + and synchronous generator functions are also now accepted, with an + asynchronous wrapper returned. + .. class:: ExitStack() @@ -564,6 +600,10 @@ Functions and classes provided: Raises :exc:`TypeError` instead of :exc:`AttributeError` if *cm* is not a context manager. + .. versionchanged:: 3.15 + Added support for arbitrary descriptors :meth:`!__enter__` and + :meth:`!__exit__`. + .. method:: push(exit) Adds a context manager's :meth:`~object.__exit__` method to the callback stack. @@ -582,6 +622,9 @@ Functions and classes provided: The passed in object is returned from the function, allowing this method to be used as a function decorator. + .. versionchanged:: 3.15 + Added support for arbitrary descriptors :meth:`!__exit__`. + .. method:: callback(callback, /, *args, **kwds) Accepts an arbitrary callback function and arguments and adds it to @@ -639,11 +682,17 @@ Functions and classes provided: Raises :exc:`TypeError` instead of :exc:`AttributeError` if *cm* is not an asynchronous context manager. + .. versionchanged:: 3.15 + Added support for arbitrary descriptors :meth:`!__aenter__` and :meth:`!__aexit__`. + .. method:: push_async_exit(exit) Similar to :meth:`ExitStack.push` but expects either an asynchronous context manager or a coroutine function. + .. versionchanged:: 3.15 + Added support for arbitrary descriptors :meth:`!__aexit__`. + .. method:: push_async_callback(callback, /, *args, **kwds) Similar to :meth:`ExitStack.callback` but expects a coroutine function. @@ -668,7 +717,7 @@ Examples and Recipes -------------------- This section describes some examples and recipes for making effective use of -the tools provided by :mod:`contextlib`. +the tools provided by :mod:`!contextlib`. Supporting a variable number of context managers @@ -697,9 +746,9 @@ context management protocol. Catching exceptions from ``__enter__`` methods ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -It is occasionally desirable to catch exceptions from an ``__enter__`` +It is occasionally desirable to catch exceptions from an :meth:`~object.__enter__` method implementation, *without* inadvertently catching exceptions from -the :keyword:`with` statement body or the context manager's ``__exit__`` +the :keyword:`with` statement body or the context manager's :meth:`~object.__exit__` method. By using :class:`ExitStack` the steps in the context management protocol can be separated slightly in order to allow this:: diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 57580ce026e..93d0c0d34bf 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -4,8 +4,6 @@ .. module:: contextvars :synopsis: Context Variables -.. sectionauthor:: Yury Selivanov - -------------- This module provides APIs to manage, store, and access context-local @@ -77,6 +75,32 @@ Context Variables to restore the variable to its previous value via the :meth:`ContextVar.reset` method. + For convenience, the token object can be used as a context manager + to avoid calling :meth:`ContextVar.reset` manually:: + + var = ContextVar('var', default='default value') + + with var.set('new value'): + assert var.get() == 'new value' + + assert var.get() == 'default value' + + It is a shorthand for:: + + var = ContextVar('var', default='default value') + + token = var.set('new value') + try: + assert var.get() == 'new value' + finally: + var.reset(token) + + assert var.get() == 'default value' + + .. versionadded:: 3.14 + + Added support for using tokens as context managers. + .. method:: reset(token) Reset the context variable to the value it had before the @@ -93,24 +117,18 @@ Context Variables # After the reset call the var has no value again, so # var.get() would raise a LookupError. + The same *token* cannot be used twice. + .. class:: Token *Token* objects are returned by the :meth:`ContextVar.set` method. They can be passed to the :meth:`ContextVar.reset` method to revert the value of the variable to what it was before the corresponding - *set*. + *set*. A single token cannot reset a context variable more than once. - The token supports :ref:`context manager protocol ` - to restore the corresponding context variable value at the exit from - :keyword:`with` block:: - - var = ContextVar('var', default='default value') - - with var.set('new value'): - assert var.get() == 'new value' - - assert var.get() == 'default value' + Tokens support the :ref:`context manager protocol ` + to automatically reset context variables. See :meth:`ContextVar.set`. .. versionadded:: 3.14 @@ -313,7 +331,7 @@ client:: addr = writer.transport.get_extra_info('socket').getpeername() client_addr_var.set(addr) - # In any code that we call is now possible to get + # In any code that we call, it is now possible to get the # client's address by calling 'client_addr_var.get()'. while True: diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 210ad718800..121c44a16ad 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -80,7 +80,7 @@ of lists by assigning a slice of the entire list, for example, Classes can use the same interfaces to control copying that they use to control pickling. See the description of module :mod:`pickle` for information on these -methods. In fact, the :mod:`copy` module uses the registered +methods. In fact, the :mod:`!copy` module uses the registered pickle functions from the :mod:`copyreg` module. .. index:: diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index 6e3144824eb..d59936029da 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -12,7 +12,7 @@ -------------- -The :mod:`copyreg` module offers a way to define functions used while pickling +The :mod:`!copyreg` module offers a way to define functions used while pickling specific objects. The :mod:`pickle` and :mod:`copy` modules use those functions when pickling/copying those objects. The module provides configuration information about object constructors which are not classes. diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst index 9ff37196ccf..647cb4925f3 100644 --- a/Doc/library/crypt.rst +++ b/Doc/library/crypt.rst @@ -13,7 +13,7 @@ being deprecated in Python 3.11. The removal was decided in :pep:`594`. Applications can use the :mod:`hashlib` module from the standard library. Other possible replacements are third-party libraries from PyPI: -:pypi:`legacycrypt`, :pypi:`bcrypt`, :pypi:`argon2-cffi`, or :pypi:`passlib`. +:pypi:`legacycrypt`, :pypi:`bcrypt`, or :pypi:`argon2-cffi`. These are not supported or maintained by the Python core team. The last version of Python that provided the :mod:`!crypt` module was diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index c11c9b8b2bf..21ecdbcc08f 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -4,8 +4,6 @@ .. module:: csv :synopsis: Write and read tabular data to and from delimited files. -.. sectionauthor:: Skip Montanaro - **Source code:** :source:`Lib/csv.py` .. index:: @@ -25,14 +23,14 @@ similar enough that it is possible to write a single module which can efficiently manipulate such data, hiding the details of reading and writing the data from the programmer. -The :mod:`csv` module implements classes to read and write tabular data in CSV +The :mod:`!csv` module implements classes to read and write tabular data in CSV format. It allows programmers to say, "write this data in the format preferred by Excel," or "read data from this file which was generated by Excel," without knowing the precise details of the CSV format used by Excel. Programmers can also describe the CSV formats understood by other applications or define their own special-purpose CSV formats. -The :mod:`csv` module's :class:`reader` and :class:`writer` objects read and +The :mod:`!csv` module's :class:`reader` and :class:`writer` objects read and write sequences. Programmers can also read and write data in dictionary form using the :class:`DictReader` and :class:`DictWriter` classes. @@ -47,7 +45,7 @@ using the :class:`DictReader` and :class:`DictWriter` classes. Module Contents --------------- -The :mod:`csv` module defines the following functions: +The :mod:`!csv` module defines the following functions: .. index:: @@ -146,7 +144,7 @@ The :mod:`csv` module defines the following functions: given, this becomes the new limit. -The :mod:`csv` module defines the following classes: +The :mod:`!csv` module defines the following classes: .. class:: DictReader(f, fieldnames=None, restkey=None, restval=None, \ dialect='excel', *args, **kwds) @@ -295,8 +293,8 @@ The :mod:`csv` module defines the following classes: - the second through n-th rows contain strings where at least one value's length differs from that of the putative header of that column. - Twenty rows after the first row are sampled; if more than half of columns + - rows meet the criteria, :const:`True` is returned. + Twenty-one rows after the header are sampled; if more than half of the + columns + rows meet the criteria, :const:`True` is returned. .. note:: @@ -314,7 +312,7 @@ An example for :class:`Sniffer` use:: .. _csv-constants: -The :mod:`csv` module defines the following constants: +The :mod:`!csv` module defines the following constants: .. data:: QUOTE_ALL @@ -375,7 +373,7 @@ The :mod:`csv` module defines the following constants: .. versionadded:: 3.12 -The :mod:`csv` module defines the following exception: +The :mod:`!csv` module defines the following exception: .. exception:: Error @@ -468,7 +466,8 @@ Dialects support the following attributes: .. attribute:: Dialect.skipinitialspace When :const:`True`, spaces immediately following the *delimiter* are ignored. - The default is :const:`False`. + The default is :const:`False`. When combining ``delimiter=' '`` with + ``skipinitialspace=True``, unquoted empty fields are not allowed. .. attribute:: Dialect.strict @@ -637,7 +636,7 @@ done:: .. rubric:: Footnotes .. [1] If ``newline=''`` is not specified, newlines embedded inside quoted fields - will not be interpreted correctly, and on platforms that use ``\r\n`` linendings + will not be interpreted correctly, and on platforms that use ``\r\n`` line endings on write an extra ``\r`` will be added. It should always be safe to specify ``newline=''``, since the csv module does its own (:term:`universal `) newline handling. diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index d8dac24c8ab..b8d615565a5 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -4,26 +4,22 @@ .. module:: ctypes :synopsis: A foreign function library for Python. -.. moduleauthor:: Thomas Heller - **Source code:** :source:`Lib/ctypes` -------------- -:mod:`ctypes` is a foreign function library for Python. It provides C compatible +:mod:`!ctypes` is a foreign function library for Python. It provides C compatible data types, and allows calling functions in DLLs or shared libraries. It can be used to wrap these libraries in pure Python. +.. include:: ../includes/optional-module.rst + .. _ctypes-ctypes-tutorial: ctypes tutorial --------------- -Note: The code samples in this tutorial use :mod:`doctest` to make sure that -they actually work. Since some code samples behave differently under Linux, -Windows, or macOS, they contain doctest directives in comments. - Note: Some code samples reference the ctypes :class:`c_int` type. On platforms where ``sizeof(long) == sizeof(int)`` it is an alias to :class:`c_long`. So, you should not be confused if :class:`c_long` is printed if you would expect @@ -34,13 +30,16 @@ So, you should not be confused if :class:`c_long` is printed if you would expect Loading dynamic link libraries ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` exports the *cdll*, and on Windows *windll* and *oledll* +:mod:`!ctypes` exports the :py:data:`~ctypes.cdll`, and on Windows +:py:data:`~ctypes.windll` and :py:data:`~ctypes.oledll` objects, for loading dynamic link libraries. -You load libraries by accessing them as attributes of these objects. *cdll* -loads libraries which export functions using the standard ``cdecl`` calling -convention, while *windll* libraries call functions using the ``stdcall`` -calling convention. *oledll* also uses the ``stdcall`` calling convention, and +You load libraries by accessing them as attributes of these objects. +:py:data:`!cdll` loads libraries which export functions using the +standard ``cdecl`` calling convention, while :py:data:`!windll` +libraries call functions using the ``stdcall`` +calling convention. +:py:data:`~oledll` also uses the ``stdcall`` calling convention, and assumes the functions return a Windows :c:type:`!HRESULT` error code. The error code is used to automatically raise an :class:`OSError` exception when the function call fails. @@ -70,11 +69,13 @@ Windows appends the usual ``.dll`` file suffix automatically. being used by Python. Where possible, use native Python functionality, or else import and use the ``msvcrt`` module. -On Linux, it is required to specify the filename *including* the extension to +Other systems require the filename *including* the extension to load a library, so attribute access can not be used to load libraries. Either the :meth:`~LibraryLoader.LoadLibrary` method of the dll loaders should be used, -or you should load the library by creating an instance of CDLL by calling -the constructor:: +or you should load the library by creating an instance of :py:class:`CDLL` +by calling the constructor. + +For example, on Linux:: >>> cdll.LoadLibrary("libc.so.6") # doctest: +LINUX @@ -83,7 +84,14 @@ the constructor:: >>> -.. XXX Add section for macOS. +On macOS:: + + >>> cdll.LoadLibrary("libc.dylib") # doctest: +MACOS + + >>> libc = CDLL("libc.dylib") # doctest: +MACOS + >>> libc # doctest: +MACOS + + .. _ctypes-accessing-functions-from-loaded-dlls: @@ -180,7 +188,7 @@ handle (passing ``None`` as single argument to call it with a ``NULL`` pointer): To find out the correct calling convention you have to look into the C header file or the documentation for the function you want to call. -On Windows, :mod:`ctypes` uses win32 structured exception handling to prevent +On Windows, :mod:`!ctypes` uses win32 structured exception handling to prevent crashes from general protection faults when functions are called with invalid argument values:: @@ -190,7 +198,7 @@ argument values:: OSError: exception: access violation reading 0x00000020 >>> -There are, however, enough ways to crash Python with :mod:`ctypes`, so you +There are, however, enough ways to crash Python with :mod:`!ctypes`, so you should be careful anyway. The :mod:`faulthandler` module can be helpful in debugging crashes (e.g. from segmentation faults produced by erroneous C library calls). @@ -203,7 +211,7 @@ as pointer to the memory block that contains their data (:c:expr:`char *` or :c:expr:`int` type, their value is masked to fit into the C type. Before we move on calling functions with other parameter types, we have to learn -more about :mod:`ctypes` data types. +more about :mod:`!ctypes` data types. .. _ctypes-fundamental-data-types: @@ -211,89 +219,166 @@ more about :mod:`ctypes` data types. Fundamental data types ^^^^^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` defines a number of primitive C compatible data types: +:mod:`!ctypes` defines a number of primitive C compatible data types: -+----------------------+------------------------------------------+----------------------------+ -| ctypes type | C type | Python type | -+======================+==========================================+============================+ -| :class:`c_bool` | :c:expr:`_Bool` | bool (1) | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_char` | :c:expr:`char` | 1-character bytes object | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_wchar` | :c:type:`wchar_t` | 1-character string | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_byte` | :c:expr:`char` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ubyte` | :c:expr:`unsigned char` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_short` | :c:expr:`short` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ushort` | :c:expr:`unsigned short` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int` | :c:expr:`int` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int8` | :c:type:`int8_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int16` | :c:type:`int16_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int32` | :c:type:`int32_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int64` | :c:type:`int64_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint` | :c:expr:`unsigned int` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint8` | :c:type:`uint8_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint16` | :c:type:`uint16_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint32` | :c:type:`uint32_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint64` | :c:type:`uint64_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_long` | :c:expr:`long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ulong` | :c:expr:`unsigned long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_longlong` | :c:expr:`__int64` or :c:expr:`long long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ulonglong` | :c:expr:`unsigned __int64` or | int | -| | :c:expr:`unsigned long long` | | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_size_t` | :c:type:`size_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ssize_t` | :c:type:`ssize_t` or | int | -| | :c:expr:`Py_ssize_t` | | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_time_t` | :c:type:`time_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_float` | :c:expr:`float` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_double` | :c:expr:`double` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_longdouble`| :c:expr:`long double` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_char_p` | :c:expr:`char *` (NUL terminated) | bytes object or ``None`` | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_wchar_p` | :c:expr:`wchar_t *` (NUL terminated) | string or ``None`` | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_void_p` | :c:expr:`void *` | int or ``None`` | -+----------------------+------------------------------------------+----------------------------+ +.. list-table:: + :header-rows: 1 -(1) - The constructor accepts any object with a truth value. + * - ctypes type + - C type + - Python type + - :py:attr:`~_SimpleCData._type_` + * - :class:`c_bool` + - :c:expr:`_Bool` + - :py:class:`bool` + - ``'?'`` + * - :class:`c_char` + - :c:expr:`char` + - 1-character :py:class:`bytes` + - ``'c'`` + * - :class:`c_wchar` + - :c:type:`wchar_t` + - 1-character :py:class:`str` + - ``'u'`` + * - :class:`c_byte` + - :c:expr:`char` + - :py:class:`int` + - ``'b'`` + * - :class:`c_ubyte` + - :c:expr:`unsigned char` + - :py:class:`int` + - ``'B'`` + * - :class:`c_short` + - :c:expr:`short` + - :py:class:`int` + - ``'h'`` + * - :class:`c_ushort` + - :c:expr:`unsigned short` + - :py:class:`int` + - ``'H'`` + * - :class:`c_int` + - :c:expr:`int` + - :py:class:`int` + - ``'i'`` \* + * - :class:`c_int8` + - :c:type:`int8_t` + - :py:class:`int` + - \* + * - :class:`c_int16` + - :c:type:`int16_t` + - :py:class:`int` + - \* + * - :class:`c_int32` + - :c:type:`int32_t` + - :py:class:`int` + - \* + * - :class:`c_int64` + - :c:type:`int64_t` + - :py:class:`int` + - \* + * - :class:`c_uint` + - :c:expr:`unsigned int` + - :py:class:`int` + - ``'I'`` \* + * - :class:`c_uint8` + - :c:type:`uint8_t` + - :py:class:`int` + - \* + * - :class:`c_uint16` + - :c:type:`uint16_t` + - :py:class:`int` + - \* + * - :class:`c_uint32` + - :c:type:`uint32_t` + - :py:class:`int` + - \* + * - :class:`c_uint64` + - :c:type:`uint64_t` + - :py:class:`int` + - \* + * - :class:`c_long` + - :c:expr:`long` + - :py:class:`int` + - ``'l'`` + * - :class:`c_ulong` + - :c:expr:`unsigned long` + - :py:class:`int` + - ``'L'`` + * - :class:`c_longlong` + - :c:expr:`long long` + - :py:class:`int` + - ``'q'`` \* + * - :class:`c_ulonglong` + - :c:expr:`unsigned long long` + - :py:class:`int` + - ``'Q'`` \* + * - :class:`c_size_t` + - :c:type:`size_t` + - :py:class:`int` + - \* + * - :class:`c_ssize_t` + - :c:type:`Py_ssize_t` + - :py:class:`int` + - \* + * - :class:`c_time_t` + - :c:type:`time_t` + - :py:class:`int` + - \* + * - :class:`c_float` + - :c:expr:`float` + - :py:class:`float` + - ``'f'`` + * - :class:`c_double` + - :c:expr:`double` + - :py:class:`float` + - ``'d'`` + * - :class:`c_longdouble` + - :c:expr:`long double` + - :py:class:`float` + - ``'g'`` \* + * - :class:`c_char_p` + - :c:expr:`char *` (NUL terminated) + - :py:class:`bytes` or ``None`` + - ``'z'`` + * - :class:`c_wchar_p` + - :c:expr:`wchar_t *` (NUL terminated) + - :py:class:`str` or ``None`` + - ``'Z'`` + * - :class:`c_void_p` + - :c:expr:`void *` + - :py:class:`int` or ``None`` + - ``'P'`` + * - :class:`py_object` + - :c:expr:`PyObject *` + - :py:class:`object` + - ``'O'`` + * - :ref:`VARIANT_BOOL ` + - :c:expr:`short int` + - :py:class:`bool` + - ``'v'`` Additionally, if IEC 60559 compatible complex arithmetic (Annex G) is supported in both C and ``libffi``, the following complex types are available: -+----------------------------------+---------------------------------+-----------------+ -| ctypes type | C type | Python type | -+==================================+=================================+=================+ -| :class:`c_float_complex` | :c:expr:`float complex` | complex | -+----------------------------------+---------------------------------+-----------------+ -| :class:`c_double_complex` | :c:expr:`double complex` | complex | -+----------------------------------+---------------------------------+-----------------+ -| :class:`c_longdouble_complex` | :c:expr:`long double complex` | complex | -+----------------------------------+---------------------------------+-----------------+ +.. list-table:: + :header-rows: 1 + + * - ctypes type + - C type + - Python type + - :py:attr:`~_SimpleCData._type_` + * - :class:`c_float_complex` + - :c:expr:`float complex` + - :py:class:`complex` + - ``'F'`` + * - :class:`c_double_complex` + - :c:expr:`double complex` + - :py:class:`complex` + - ``'D'`` + * - :class:`c_longdouble_complex` + - :c:expr:`long double complex` + - :py:class:`complex` + - ``'G'`` All these types can be created by calling them with an optional initializer of @@ -307,6 +392,16 @@ the correct type and value:: c_ushort(65533) >>> +The constructors for numeric types will convert input using +:py:meth:`~object.__bool__`, +:py:meth:`~object.__index__` (for ``int``), +:py:meth:`~object.__float__` or :py:meth:`~object.__complex__`. +This means :py:class:`~ctypes.c_bool` accepts any object with a truth value:: + + >>> empty_list = [] + >>> c_bool(empty_list) + c_bool(False) + Since these types are mutable, their value can also be changed afterwards:: >>> i = c_int(42) @@ -395,7 +490,7 @@ from within *IDLE* or *PythonWin*:: >>> As has been mentioned before, all Python types except integers, strings, and -bytes objects have to be wrapped in their corresponding :mod:`ctypes` type, so +bytes objects have to be wrapped in their corresponding :mod:`!ctypes` type, so that they can be converted to the required C data type:: >>> printf(b"An int %d, a double %f\n", 1234, c_double(3.14)) @@ -429,10 +524,10 @@ specify :attr:`~_CFuncPtr.argtypes` for all variadic functions. Calling functions with your own custom data types ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -You can also customize :mod:`ctypes` argument conversion to allow instances of -your own classes be used as function arguments. :mod:`ctypes` looks for an +You can also customize :mod:`!ctypes` argument conversion to allow instances of +your own classes be used as function arguments. :mod:`!ctypes` looks for an :attr:`!_as_parameter_` attribute and uses this as the function argument. The -attribute must be an integer, string, bytes, a :mod:`ctypes` instance, or an +attribute must be an integer, string, bytes, a :mod:`!ctypes` instance, or an object with an :attr:`!_as_parameter_` attribute:: >>> class Bottles: @@ -488,7 +583,7 @@ the Python object passed to the function call, it should do a typecheck or whatever is needed to make sure this object is acceptable, and then return the object itself, its :attr:`!_as_parameter_` attribute, or whatever you want to pass as the C function argument in this case. Again, the result should be an -integer, string, bytes, a :mod:`ctypes` instance, or an object with an +integer, string, bytes, a :mod:`!ctypes` instance, or an object with an :attr:`!_as_parameter_` attribute. @@ -598,7 +693,7 @@ Sometimes a C api function expects a *pointer* to a data type as parameter, probably to write into the corresponding location, or if the data is too large to be passed by value. This is also known as *passing parameters by reference*. -:mod:`ctypes` exports the :func:`byref` function which is used to pass parameters +:mod:`!ctypes` exports the :func:`byref` function which is used to pass parameters by reference. The same effect can be achieved with the :func:`pointer` function, although :func:`pointer` does a lot more work since it constructs a real pointer object, so it is faster to use :func:`byref` if you don't need the pointer @@ -623,12 +718,12 @@ Structures and unions ^^^^^^^^^^^^^^^^^^^^^ Structures and unions must derive from the :class:`Structure` and :class:`Union` -base classes which are defined in the :mod:`ctypes` module. Each subclass must +base classes which are defined in the :mod:`!ctypes` module. Each subclass must define a :attr:`~Structure._fields_` attribute. :attr:`!_fields_` must be a list of *2-tuples*, containing a *field name* and a *field type*. -The field type must be a :mod:`ctypes` type like :class:`c_int`, or any other -derived :mod:`ctypes` type: structure, union, array, pointer. +The field type must be a :mod:`!ctypes` type like :class:`c_int`, or any other +derived :mod:`!ctypes` type: structure, union, array, pointer. Here is a simple example of a POINT structure, which contains two integers named *x* and *y*, and also shows how to initialize a structure in the constructor:: @@ -687,7 +782,7 @@ See :class:`CField`:: .. warning:: - :mod:`ctypes` does not support passing unions or structures with bit-fields + :mod:`!ctypes` does not support passing unions or structures with bit-fields to functions by value. While this may work on 32-bit x86, it's not guaranteed by the library to work in the general case. Unions and structures with bit-fields should always be passed to functions by pointer. @@ -705,7 +800,7 @@ structure itself by setting the class attributes :attr:`~Structure._pack_` and/or :attr:`~Structure._align_`, respectively. See the attribute documentation for details. -:mod:`ctypes` uses the native byte order for Structures and Unions. To build +:mod:`!ctypes` uses the native byte order for Structures and Unions. To build structures with non-native byte order, you can use one of the :class:`BigEndianStructure`, :class:`LittleEndianStructure`, :class:`BigEndianUnion`, and :class:`LittleEndianUnion` base classes. These @@ -794,7 +889,7 @@ Pointers ^^^^^^^^ Pointer instances are created by calling the :func:`pointer` function on a -:mod:`ctypes` type:: +:mod:`!ctypes` type:: >>> from ctypes import * >>> i = c_int(42) @@ -808,7 +903,7 @@ returns the object to which the pointer points, the ``i`` object above:: c_long(42) >>> -Note that :mod:`ctypes` does not have OOR (original object return), it constructs a +Note that :mod:`!ctypes` does not have OOR (original object return), it constructs a new, equivalent object each time you retrieve an attribute:: >>> pi.contents is i @@ -852,7 +947,7 @@ item. Behind the scenes, the :func:`pointer` function does more than simply create pointer instances, it has to create pointer *types* first. This is done with the -:func:`POINTER` function, which accepts any :mod:`ctypes` type, and returns a +:func:`POINTER` function, which accepts any :mod:`!ctypes` type, and returns a new type:: >>> PI = POINTER(c_int) @@ -874,7 +969,7 @@ Calling the pointer type without an argument creates a ``NULL`` pointer. False >>> -:mod:`ctypes` checks for ``NULL`` when dereferencing pointers (but dereferencing +:mod:`!ctypes` checks for ``NULL`` when dereferencing pointers (but dereferencing invalid non-\ ``NULL`` pointers would crash Python):: >>> null_ptr[0] @@ -894,7 +989,7 @@ invalid non-\ ``NULL`` pointers would crash Python):: Thread safety without the GIL ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -From Python 3.13 onward, the :term:`GIL` can be disabled on :term:`free threaded ` builds. +From Python 3.13 onward, the :term:`GIL` can be disabled on the :term:`free-threaded build`. In ctypes, reads and writes to a single object concurrently is safe, but not across multiple objects: .. code-block:: pycon @@ -959,7 +1054,7 @@ To set a POINTER type field to ``NULL``, you can assign ``None``:: .. XXX list other conversions... Sometimes you have instances of incompatible types. In C, you can cast one type -into another type. :mod:`ctypes` provides a :func:`cast` function which can be +into another type. :mod:`!ctypes` provides a :func:`cast` function which can be used in the same way. The ``Bar`` structure defined above accepts ``POINTER(c_int)`` pointers or :class:`c_int` arrays for its ``values`` field, but not instances of other types:: @@ -1023,7 +1118,7 @@ work:: >>> because the new ``class cell`` is not available in the class statement itself. -In :mod:`ctypes`, we can define the ``cell`` class and set the +In :mod:`!ctypes`, we can define the ``cell`` class and set the :attr:`~Structure._fields_` attribute later, after the class statement:: >>> from ctypes import * @@ -1057,7 +1152,7 @@ other, and finally follow the pointer chain a few times:: Callback functions ^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` allows creating C callable function pointers from Python callables. +:mod:`!ctypes` allows creating C callable function pointers from Python callables. These are sometimes called *callback functions*. First, you must create a class for the callback function. The class knows the @@ -1156,7 +1251,7 @@ write:: .. note:: Make sure you keep references to :func:`CFUNCTYPE` objects as long as they - are used from C code. :mod:`ctypes` doesn't, and if you don't, they may be + are used from C code. :mod:`!ctypes` doesn't, and if you don't, they may be garbage collected, crashing your program when a callback is made. Also, note that if the callback function is called in a thread created @@ -1175,7 +1270,7 @@ Some shared libraries not only export functions, they also export variables. An example in the Python library itself is the :c:data:`Py_Version`, Python runtime version number encoded in a single constant integer. -:mod:`ctypes` can access values like this with the :meth:`~_CData.in_dll` class methods of +:mod:`!ctypes` can access values like this with the :meth:`~_CData.in_dll` class methods of the type. *pythonapi* is a predefined symbol giving access to the Python C api:: @@ -1194,7 +1289,7 @@ Quoting the docs for that value: tricks with this to provide a dynamically created collection of frozen modules. So manipulating this pointer could even prove useful. To restrict the example -size, we show only how this table can be read with :mod:`ctypes`:: +size, we show only how this table can be read with :mod:`!ctypes`:: >>> from ctypes import * >>> @@ -1240,7 +1335,7 @@ for testing. Try it out with ``import __hello__`` for example. Surprises ^^^^^^^^^ -There are some edges in :mod:`ctypes` where you might expect something other +There are some edges in :mod:`!ctypes` where you might expect something other than what actually happens. Consider the following example:: @@ -1308,7 +1403,7 @@ constructs a new Python object each time! Variable-sized data types ^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` provides some support for variable-sized arrays and structures. +:mod:`!ctypes` provides some support for variable-sized arrays and structures. The :func:`resize` function can be used to resize the memory buffer of an existing ctypes object. The function takes the object as first argument, and @@ -1342,7 +1437,7 @@ get errors accessing other elements:: IndexError: invalid index >>> -Another way to use variable-sized data types with :mod:`ctypes` is to use the +Another way to use variable-sized data types with :mod:`!ctypes` is to use the dynamic nature of Python, and (re-)define the data type after the required size is already known, on a case by case basis. @@ -1352,6 +1447,271 @@ is already known, on a case by case basis. ctypes reference ---------------- +.. _ctypes-loading-shared-libraries: + +Loading shared libraries +^^^^^^^^^^^^^^^^^^^^^^^^ + +There are several ways to load shared libraries into the Python process. One +way is to instantiate :py:class:`CDLL` or one of its subclasses: + + +.. class:: CDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) + + Represents a loaded shared library. + + Functions in this library use the standard C calling convention, and are + assumed to return :c:expr:`int`. + The Python :term:`global interpreter lock` is released before calling any + function exported by these libraries, and reacquired afterwards. + For different function behavior, use a subclass: :py:class:`~ctypes.OleDLL`, + :py:class:`~ctypes.WinDLL`, or :py:class:`~ctypes.PyDLL`. + + If you have an existing :py:attr:`handle ` to an already + loaded shared library, it can be passed as the *handle* argument to wrap + the opened library in a new :py:class:`!CDLL` object. + In this case, *name* is only used to set the :py:attr:`~ctypes.CDLL._name` + attribute, but it may be adjusted and/or validated. + + If *handle* is ``None``, the underlying platform's :manpage:`dlopen(3)` or + `LoadLibraryExW`_ function is used to load the library into + the process, and to get a handle to it. + + *name* is the pathname of the shared library to open. + If *name* does not contain a path separator, the library is found + in a platform-specific way. + + On Windows, the ``.DLL`` suffix may be missing. (For details, see + `LoadLibraryExW`_ documentation.) + Other platform-specific prefixes and suffixes (for example, ``lib``, + ``.so``, ``.dylib``, or version numbers) must be present in *name*; + they are not added automatically. + See :ref:`ctypes-finding-shared-libraries` for more information. + + On non-Windows systems, *name* can be ``None``. In this case, + :c:func:`!dlopen` is called with ``NULL``, which opens the main program + as a "library". + (Some systems do the same is *name* is empty; ``None``/``NULL`` is more + portable.) + + .. admonition:: CPython implementation detail + + Since CPython is linked to ``libc``, a ``None`` *name* is often used + to access the C standard library:: + + >>> printf = ctypes.CDLL(None).printf + >>> printf.argtypes = [ctypes.c_char_p] + >>> printf(b"hello\n") + hello + 6 + + To access the Python C API, prefer :py:data:`ctypes.pythonapi` which + works across platforms. + + The *mode* parameter can be used to specify how the library is loaded. For + details, consult the :manpage:`dlopen(3)` manpage. On Windows, *mode* is + ignored. On posix systems, RTLD_NOW is always added, and is not + configurable. + + The *use_errno* parameter, when set to true, enables a ctypes mechanism that + allows accessing the system :data:`errno` error number in a safe way. + :mod:`!ctypes` maintains a thread-local copy of the system's :data:`errno` + variable; if you call foreign functions created with ``use_errno=True`` then the + :data:`errno` value before the function call is swapped with the ctypes private + copy, the same happens immediately after the function call. + + The function :func:`ctypes.get_errno` returns the value of the ctypes private + copy, and the function :func:`ctypes.set_errno` changes the ctypes private copy + to a new value and returns the former value. + + The *use_last_error* parameter, when set to true, enables the same mechanism for + the Windows error code which is managed by the :func:`GetLastError` and + :func:`!SetLastError` Windows API functions; :func:`ctypes.get_last_error` and + :func:`ctypes.set_last_error` are used to request and change the ctypes private + copy of the windows error code. + + The *winmode* parameter is used on Windows to specify how the library is loaded + (since *mode* is ignored). It takes any value that is valid for the Win32 API + `LoadLibraryExW`_ flags parameter. When omitted, the default is to use the + flags that result in the most secure DLL load, which avoids issues such as DLL + hijacking. Passing the full path to the DLL is the safest way to ensure the + correct library and dependencies are loaded. + + On Windows creating a :class:`CDLL` instance may fail even if the DLL name + exists. When a dependent DLL of the loaded DLL is not found, a + :exc:`OSError` error is raised with the message *"[WinError 126] The + specified module could not be found".* This error message does not contain + the name of the missing DLL because the Windows API does not return this + information making this error hard to diagnose. To resolve this error and + determine which DLL is not found, you need to find the list of dependent + DLLs and determine which one is not found using Windows debugging and + tracing tools. + + .. seealso:: + + `Microsoft DUMPBIN tool `_ + -- A tool to find DLL dependents. + + .. versionchanged:: 3.8 + Added *winmode* parameter. + + .. versionchanged:: 3.12 + + The *name* parameter can now be a :term:`path-like object`. + + Instances of this class have no public methods. Functions exported by the + shared library can be accessed as attributes or by index. Please note that + accessing the function through an attribute caches the result and therefore + accessing it repeatedly returns the same object each time. On the other hand, + accessing it through an index returns a new object each time:: + + >>> from ctypes import CDLL + >>> libc = CDLL("libc.so.6") # On Linux + >>> libc.time == libc.time + True + >>> libc['time'] == libc['time'] + False + + The following public attributes are available. Their name starts with an + underscore to not clash with exported function names: + + .. attribute:: _handle + + The system handle used to access the library. + + .. attribute:: _name + + The name of the library passed in the constructor. + +.. _LoadLibraryExW: https://learn.microsoft.com/en-us/windows/win32/api/libloaderapi/nf-libloaderapi-loadlibraryexw + +.. class:: OleDLL + + See :py:class:`~ctypes.CDLL`, the superclass, for common information. + + Functions in this library use the ``stdcall`` calling convention, and are + assumed to return the windows specific :class:`HRESULT` code. :class:`HRESULT` + values contain information specifying whether the function call failed or + succeeded, together with additional error code. If the return value signals a + failure, an :class:`OSError` is automatically raised. + + .. availability:: Windows + + .. versionchanged:: 3.3 + :exc:`WindowsError` used to be raised, + which is now an alias of :exc:`OSError`. + + +.. class:: WinDLL + + See :py:class:`~ctypes.CDLL`, the superclass, for common information. + + Functions in these libraries use the ``stdcall`` calling convention, and are + assumed to return :c:expr:`int` by default. + + .. availability:: Windows + +.. class:: PyDLL + + See :py:class:`~ctypes.CDLL`, the superclass, for common information. + + When functions in this library are called, the + Python GIL is *not* released during the function call, and after the function + execution the Python error flag is checked. If the error flag is set, a Python + exception is raised. + + Thus, this is only useful to call Python C API functions directly. + + +.. data:: RTLD_GLOBAL + + Flag to use as *mode* parameter. On platforms where this flag is not available, + it is defined as the integer zero. + + +.. data:: RTLD_LOCAL + + Flag to use as *mode* parameter. On platforms where this is not available, it + is the same as *RTLD_GLOBAL*. + + +.. data:: DEFAULT_MODE + + The default mode which is used to load shared libraries. On OSX 10.3, this is + *RTLD_GLOBAL*, otherwise it is the same as *RTLD_LOCAL*. + + +Shared libraries can also be loaded by using one of the prefabricated objects, +which are instances of the :class:`LibraryLoader` class, either by calling the +:meth:`~LibraryLoader.LoadLibrary` method, or by retrieving the library as +attribute of the loader instance. + +.. class:: LibraryLoader(dlltype) + + Class which loads shared libraries. *dlltype* should be one of the + :class:`CDLL`, :class:`PyDLL`, :class:`WinDLL`, or :class:`OleDLL` types. + + :meth:`!__getattr__` has special behavior: It allows loading a shared library by + accessing it as attribute of a library loader instance. The result is cached, + so repeated attribute accesses return the same library each time. + + .. method:: LoadLibrary(name) + + Load a shared library into the process and return it. This method always + returns a new instance of the library. + + +These prefabricated library loaders are available: + + .. data:: cdll + + Creates :class:`CDLL` instances. + + + .. data:: windll + + Creates :class:`WinDLL` instances. + + .. availability:: Windows + + + .. data:: oledll + + Creates :class:`OleDLL` instances. + + .. availability:: Windows + + + .. data:: pydll + + Creates :class:`PyDLL` instances. + + + .. data:: pythonapi + + An instance of :class:`PyDLL` that exposes Python C API functions as + attributes. Note that all these functions are assumed to return C + :c:expr:`int`, which is of course not always the truth, so you have to assign + the correct :attr:`!restype` attribute to use these functions. + +.. audit-event:: ctypes.dlopen name ctypes.LibraryLoader + + Loading a library through any of these objects raises an + :ref:`auditing event ` ``ctypes.dlopen`` with string argument + ``name``, the name used to load the library. + +.. audit-event:: ctypes.dlsym library,name ctypes.LibraryLoader + + Accessing a function on a loaded library raises an auditing event + ``ctypes.dlsym`` with arguments ``library`` (the library object) and ``name`` + (the symbol's name as a string or integer). + +.. audit-event:: ctypes.dlsym/handle handle,name ctypes.LibraryLoader + + In cases when only the library handle is available rather than the object, + accessing a function raises an auditing event ``ctypes.dlsym/handle`` with + arguments ``handle`` (the raw library handle) and ``name``. + .. _ctypes-finding-shared-libraries: @@ -1360,31 +1720,52 @@ Finding shared libraries When programming in a compiled language, shared libraries are accessed when compiling/linking a program, and when the program is run. +The programmer specifies a short name; the C compiler, linker, and +runtime dynamic library loader then interact in system-specific ways to find +the filename of the library to load. -The purpose of the :func:`~ctypes.util.find_library` function is to locate a library in a way -similar to what the compiler or runtime loader does (on platforms with several -versions of a shared library the most recent should be loaded), while the ctypes -library loaders act like when a program is run, and call the runtime loader -directly. +While the mapping from short names to filenames is not consistently exposed +by platforms, the :mod:`!ctypes.util` module provides a function, +:func:`!find_library`, that attempts to match it. +However, as backwards compatibility concerns make it difficult to adjust +its behavior for new platforms and configurations, the function +is :term:`soft deprecated`. -The :mod:`!ctypes.util` module provides a function which can help to determine -the library to load. +If wrapping a shared library with :mod:`!ctypes`, consider determining the +shared library name at development time, and hardcoding it into the wrapper +module instead of using :func:`!find_library` to locate the library +at runtime. +Also consider adding a configuration option or environment variable to let +users select a library to use, and then perhaps use :func:`!find_library` +as a default or fallback. - -.. data:: find_library(name) +.. function:: find_library(name) :module: ctypes.util - :noindex: - Try to find a library and return a pathname. *name* is the library name without - any prefix like *lib*, suffix like ``.so``, ``.dylib`` or version number (this - is the form used for the posix linker option :option:`!-l`). If no library can - be found, returns ``None``. + Try to find a library and return a pathname. -The exact functionality is system dependent. + *name* is the "short" library name without any prefix like ``lib``, + suffix like ``.so``, ``.dylib`` or version number. + (This is the form used for the posix linker option :option:`!-l`.) + The result is in a format suitable for passing to :py:class:`~ctypes.CDLL`. -On Linux, :func:`~ctypes.util.find_library` tries to run external programs -(``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the library file. -It returns the filename of the library file. + If no library can be found, return ``None``. + + The exact functionality is system dependent, and is *not guaranteed* + to match the behavior of the compiler, linker, and loader used for + (or by) Python. + It is recommended to only use this function as a default or fallback, + + .. soft-deprecated:: 3.15 + + This function is kept for use in cases where it works, but not expected to + be updated for additional platforms and configurations. + +On Linux, :func:`!find_library` tries to run external +programs (``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the +library file. +If the output of these programs does not correspond to the dynamic +linker used by Python, the result of this function may be misleading. .. versionchanged:: 3.6 On Linux, the value of the environment variable ``LD_LIBRARY_PATH`` is used @@ -1401,7 +1782,7 @@ Here are some examples:: 'libbz2.so.1.0' >>> -On macOS and Android, :func:`~ctypes.util.find_library` uses the system's +On macOS and Android, :func:`!find_library` uses the system's standard naming schemes and paths to locate the library, and returns a full pathname if successful:: @@ -1416,13 +1797,25 @@ pathname if successful:: '/System/Library/Frameworks/AGL.framework/AGL' >>> -On Windows, :func:`~ctypes.util.find_library` searches along the system search path, and +On Windows, :func:`!find_library` searches along the system search path, and returns the full pathname, but since there is no predefined naming scheme a call like ``find_library("c")`` will fail and return ``None``. -If wrapping a shared library with :mod:`ctypes`, it *may* be better to determine -the shared library name at development time, and hardcode that into the wrapper -module instead of using :func:`~ctypes.util.find_library` to locate the library at runtime. +.. function:: find_msvcrt() + :module: ctypes.util + + Returns the filename of the VC runtime library used by Python, + and by the extension modules. + + If the name of the library cannot be determined, ``None`` is returned. + Notably, this will happen for recent versions of the VC runtime library, + which are not directly loadable. + + If you need to free memory, for example, allocated by an extension module + with a call to the ``free(void *)``, it is important that you use the + function in the same library that allocated the memory. + + .. availability:: Windows .. _ctypes-listing-loaded-shared-libraries: @@ -1447,257 +1840,6 @@ For example, on glibc-based Linux, the return may look like:: >>> dllist() ['', 'linux-vdso.so.1', '/lib/x86_64-linux-gnu/libm.so.6', '/lib/x86_64-linux-gnu/libc.so.6', ... ] -.. _ctypes-loading-shared-libraries: - -Loading shared libraries -^^^^^^^^^^^^^^^^^^^^^^^^ - -There are several ways to load shared libraries into the Python process. One -way is to instantiate one of the following classes: - - -.. class:: CDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) - - Instances of this class represent loaded shared libraries. Functions in these - libraries use the standard C calling convention, and are assumed to return - :c:expr:`int`. - - On Windows creating a :class:`CDLL` instance may fail even if the DLL name - exists. When a dependent DLL of the loaded DLL is not found, a - :exc:`OSError` error is raised with the message *"[WinError 126] The - specified module could not be found".* This error message does not contain - the name of the missing DLL because the Windows API does not return this - information making this error hard to diagnose. To resolve this error and - determine which DLL is not found, you need to find the list of dependent - DLLs and determine which one is not found using Windows debugging and - tracing tools. - - .. versionchanged:: 3.12 - - The *name* parameter can now be a :term:`path-like object`. - -.. seealso:: - - `Microsoft DUMPBIN tool `_ - -- A tool to find DLL dependents. - - -.. class:: OleDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) - - Instances of this class represent loaded shared libraries, - functions in these libraries use the ``stdcall`` calling convention, and are - assumed to return the windows specific :class:`HRESULT` code. :class:`HRESULT` - values contain information specifying whether the function call failed or - succeeded, together with additional error code. If the return value signals a - failure, an :class:`OSError` is automatically raised. - - .. availability:: Windows - - .. versionchanged:: 3.3 - :exc:`WindowsError` used to be raised, - which is now an alias of :exc:`OSError`. - - .. versionchanged:: 3.12 - - The *name* parameter can now be a :term:`path-like object`. - - -.. class:: WinDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) - - Instances of this class represent loaded shared libraries, - functions in these libraries use the ``stdcall`` calling convention, and are - assumed to return :c:expr:`int` by default. - - .. availability:: Windows - - .. versionchanged:: 3.12 - - The *name* parameter can now be a :term:`path-like object`. - - -The Python :term:`global interpreter lock` is released before calling any -function exported by these libraries, and reacquired afterwards. - - -.. class:: PyDLL(name, mode=DEFAULT_MODE, handle=None) - - Instances of this class behave like :class:`CDLL` instances, except that the - Python GIL is *not* released during the function call, and after the function - execution the Python error flag is checked. If the error flag is set, a Python - exception is raised. - - Thus, this is only useful to call Python C api functions directly. - - .. versionchanged:: 3.12 - - The *name* parameter can now be a :term:`path-like object`. - -All these classes can be instantiated by calling them with at least one -argument, the pathname of the shared library. If you have an existing handle to -an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platform's :c:func:`!dlopen` or -:c:func:`!LoadLibrary` function is used to load the library into -the process, and to get a handle to it. - -The *mode* parameter can be used to specify how the library is loaded. For -details, consult the :manpage:`dlopen(3)` manpage. On Windows, *mode* is -ignored. On posix systems, RTLD_NOW is always added, and is not -configurable. - -The *use_errno* parameter, when set to true, enables a ctypes mechanism that -allows accessing the system :data:`errno` error number in a safe way. -:mod:`ctypes` maintains a thread-local copy of the system's :data:`errno` -variable; if you call foreign functions created with ``use_errno=True`` then the -:data:`errno` value before the function call is swapped with the ctypes private -copy, the same happens immediately after the function call. - -The function :func:`ctypes.get_errno` returns the value of the ctypes private -copy, and the function :func:`ctypes.set_errno` changes the ctypes private copy -to a new value and returns the former value. - -The *use_last_error* parameter, when set to true, enables the same mechanism for -the Windows error code which is managed by the :func:`GetLastError` and -:func:`!SetLastError` Windows API functions; :func:`ctypes.get_last_error` and -:func:`ctypes.set_last_error` are used to request and change the ctypes private -copy of the windows error code. - -The *winmode* parameter is used on Windows to specify how the library is loaded -(since *mode* is ignored). It takes any value that is valid for the Win32 API -``LoadLibraryEx`` flags parameter. When omitted, the default is to use the -flags that result in the most secure DLL load, which avoids issues such as DLL -hijacking. Passing the full path to the DLL is the safest way to ensure the -correct library and dependencies are loaded. - -.. versionchanged:: 3.8 - Added *winmode* parameter. - - -.. data:: RTLD_GLOBAL - :noindex: - - Flag to use as *mode* parameter. On platforms where this flag is not available, - it is defined as the integer zero. - - -.. data:: RTLD_LOCAL - :noindex: - - Flag to use as *mode* parameter. On platforms where this is not available, it - is the same as *RTLD_GLOBAL*. - - -.. data:: DEFAULT_MODE - :noindex: - - The default mode which is used to load shared libraries. On OSX 10.3, this is - *RTLD_GLOBAL*, otherwise it is the same as *RTLD_LOCAL*. - -Instances of these classes have no public methods. Functions exported by the -shared library can be accessed as attributes or by index. Please note that -accessing the function through an attribute caches the result and therefore -accessing it repeatedly returns the same object each time. On the other hand, -accessing it through an index returns a new object each time:: - - >>> from ctypes import CDLL - >>> libc = CDLL("libc.so.6") # On Linux - >>> libc.time == libc.time - True - >>> libc['time'] == libc['time'] - False - -The following public attributes are available, their name starts with an -underscore to not clash with exported function names: - - -.. attribute:: PyDLL._handle - - The system handle used to access the library. - - -.. attribute:: PyDLL._name - - The name of the library passed in the constructor. - -Shared libraries can also be loaded by using one of the prefabricated objects, -which are instances of the :class:`LibraryLoader` class, either by calling the -:meth:`~LibraryLoader.LoadLibrary` method, or by retrieving the library as -attribute of the loader instance. - - -.. class:: LibraryLoader(dlltype) - - Class which loads shared libraries. *dlltype* should be one of the - :class:`CDLL`, :class:`PyDLL`, :class:`WinDLL`, or :class:`OleDLL` types. - - :meth:`!__getattr__` has special behavior: It allows loading a shared library by - accessing it as attribute of a library loader instance. The result is cached, - so repeated attribute accesses return the same library each time. - - .. method:: LoadLibrary(name) - - Load a shared library into the process and return it. This method always - returns a new instance of the library. - - -These prefabricated library loaders are available: - -.. data:: cdll - :noindex: - - Creates :class:`CDLL` instances. - - -.. data:: windll - :noindex: - - Creates :class:`WinDLL` instances. - - .. availability:: Windows - - -.. data:: oledll - :noindex: - - Creates :class:`OleDLL` instances. - - .. availability:: Windows - - -.. data:: pydll - :noindex: - - Creates :class:`PyDLL` instances. - - -For accessing the C Python api directly, a ready-to-use Python shared library -object is available: - -.. data:: pythonapi - :noindex: - - An instance of :class:`PyDLL` that exposes Python C API functions as - attributes. Note that all these functions are assumed to return C - :c:expr:`int`, which is of course not always the truth, so you have to assign - the correct :attr:`!restype` attribute to use these functions. - -.. audit-event:: ctypes.dlopen name ctypes.LibraryLoader - - Loading a library through any of these objects raises an - :ref:`auditing event ` ``ctypes.dlopen`` with string argument - ``name``, the name used to load the library. - -.. audit-event:: ctypes.dlsym library,name ctypes.LibraryLoader - - Accessing a function on a loaded library raises an auditing event - ``ctypes.dlsym`` with arguments ``library`` (the library object) and ``name`` - (the symbol's name as a string or integer). - -.. audit-event:: ctypes.dlsym/handle handle,name ctypes.LibraryLoader - - In cases when only the library handle is available rather than the object, - accessing a function raises an auditing event ``ctypes.dlsym/handle`` with - arguments ``handle`` (the raw library handle) and ``name``. - .. _ctypes-foreign-functions: Foreign functions @@ -1924,7 +2066,7 @@ the windows header file is this:: LPCWSTR lpCaption, UINT uType); -Here is the wrapping with :mod:`ctypes`:: +Here is the wrapping with :mod:`!ctypes`:: >>> from ctypes import c_int, WINFUNCTYPE, windll >>> from ctypes.wintypes import HWND, LPCWSTR, UINT @@ -1947,7 +2089,7 @@ function retrieves the dimensions of a specified window by copying them into HWND hWnd, LPRECT lpRect); -Here is the wrapping with :mod:`ctypes`:: +Here is the wrapping with :mod:`!ctypes`:: >>> from ctypes import POINTER, WINFUNCTYPE, windll, WinError >>> from ctypes.wintypes import BOOL, HWND, RECT @@ -1975,7 +2117,7 @@ do the error checking, and raises an exception when the api call failed:: >>> If the :attr:`~_CFuncPtr.errcheck` function returns the argument tuple it receives -unchanged, :mod:`ctypes` continues the normal processing it does on the output +unchanged, :mod:`!ctypes` continues the normal processing it does on the output parameters. If you want to return a tuple of window coordinates instead of a ``RECT`` instance, you can retrieve the fields in the function and return them instead, the normal processing will no longer take place:: @@ -2120,31 +2262,6 @@ Utility functions .. availability:: Windows -.. function:: find_library(name) - :module: ctypes.util - - Try to find a library and return a pathname. *name* is the library name - without any prefix like ``lib``, suffix like ``.so``, ``.dylib`` or version - number (this is the form used for the posix linker option :option:`!-l`). If - no library can be found, returns ``None``. - - The exact functionality is system dependent. - - -.. function:: find_msvcrt() - :module: ctypes.util - - Returns the filename of the VC runtime library used by Python, - and by the extension modules. If the name of the library cannot be - determined, ``None`` is returned. - - If you need to free memory, for example, allocated by an extension module - with a call to the ``free(void *)``, it is important that you use the - function in the same library that allocated the memory. - - .. availability:: Windows - - .. function:: dllist() :module: ctypes.util @@ -2443,10 +2560,33 @@ Fundamental data types Python bytes object or string. When the ``value`` attribute is retrieved from a ctypes instance, usually - a new object is returned each time. :mod:`ctypes` does *not* implement + a new object is returned each time. :mod:`!ctypes` does *not* implement original object return, always a new object is constructed. The same is true for all other ctypes object instances. + Each subclass has a class attribute: + + .. attribute:: _type_ + + Class attribute that contains an internal type code, as a + single-character string. + See :ref:`ctypes-fundamental-data-types` for a summary. + + Types marked \* in the summary may be (or always are) aliases of a + different :class:`_SimpleCData` subclass, and will not necessarily + use the listed type code. + For example, if the platform's :c:expr:`long`, :c:expr:`long long` + and :c:expr:`time_t` C types are the same, then :class:`c_long`, + :class:`c_longlong` and :class:`c_time_t` all refer to a single class, + :class:`c_long`, whose :attr:`_type_` code is ``'l'``. + The ``'L'`` code will be unused. + + .. seealso:: + + The :mod:`array` and :ref:`struct ` modules, + as well as third-party modules like `numpy `__, + use similar -- but slightly different -- type codes. + Fundamental data types, when returned as foreign function call results, or, for example, by retrieving structure field members or array items, are transparently @@ -2568,6 +2708,8 @@ These are the fundamental ctypes data types: Represents the C :c:expr:`signed long long` datatype. The constructor accepts an optional integer initializer; no overflow checking is done. + On platforms where ``sizeof(long long) == sizeof(long)`` it is an alias + to :class:`c_long`. .. class:: c_short @@ -2579,11 +2721,15 @@ These are the fundamental ctypes data types: .. class:: c_size_t Represents the C :c:type:`size_t` datatype. + Usually an alias for another unsigned integer type. .. class:: c_ssize_t - Represents the C :c:type:`ssize_t` datatype. + Represents the :c:type:`Py_ssize_t` datatype. + This is a signed version of :c:type:`size_t`; + that is, the POSIX :c:type:`ssize_t` type. + Usually an alias for another integer type. .. versionadded:: 3.2 @@ -2591,6 +2737,7 @@ These are the fundamental ctypes data types: .. class:: c_time_t Represents the C :c:type:`time_t` datatype. + Usually an alias for another integer type. .. versionadded:: 3.12 @@ -2643,6 +2790,8 @@ These are the fundamental ctypes data types: Represents the C :c:expr:`unsigned long long` datatype. The constructor accepts an optional integer initializer; no overflow checking is done. + On platforms where ``sizeof(long long) == sizeof(long)`` it is an alias + to :class:`c_long`. .. class:: c_ushort @@ -2694,8 +2843,11 @@ These are the fundamental ctypes data types: .. versionchanged:: 3.14 :class:`!py_object` is now a :term:`generic type`. +.. _ctypes-wintypes: + The :mod:`!ctypes.wintypes` module provides quite some other Windows specific -data types, for example :c:type:`!HWND`, :c:type:`!WPARAM`, or :c:type:`!DWORD`. +data types, for example :c:type:`!HWND`, :c:type:`!WPARAM`, +:c:type:`!VARIANT_BOOL` or :c:type:`!DWORD`. Some useful structures like :c:type:`!MSG` or :c:type:`!RECT` are also defined. @@ -2742,7 +2894,7 @@ fields, or any other data types containing pointer type fields. Abstract base class for structures in *native* byte order. Concrete structure and union types must be created by subclassing one of these - types, and at least define a :attr:`_fields_` class variable. :mod:`ctypes` will + types, and at least define a :attr:`_fields_` class variable. :mod:`!ctypes` will create :term:`descriptor`\s which allow reading and writing the fields by direct attribute accesses. These are the @@ -2796,7 +2948,7 @@ fields, or any other data types containing pointer type fields. Setting :attr:`!_pack_` to 0 is the same as not setting it at all. Otherwise, the value must be a positive power of two. The effect is equivalent to ``#pragma pack(N)`` in C, except - :mod:`ctypes` may allow larger *n* than what the compiler accepts. + :mod:`!ctypes` may allow larger *n* than what the compiler accepts. :attr:`!_pack_` must already be defined when :attr:`_fields_` is assigned, otherwise it will have no effect. @@ -2817,7 +2969,7 @@ fields, or any other data types containing pointer type fields. The value must not be negative. The effect is equivalent to ``__attribute__((aligned(N)))`` on GCC - or ``#pragma align(N)`` on MSVC, except :mod:`ctypes` may allow + or ``#pragma align(N)`` on MSVC, except :mod:`!ctypes` may allow values that the compiler would reject. :attr:`!_align_` can only *increase* a structure's alignment @@ -2866,7 +3018,7 @@ fields, or any other data types containing pointer type fields. assigned, otherwise it will have no effect. The fields listed in this variable must be structure or union type fields. - :mod:`ctypes` will create descriptors in the structure type that allows + :mod:`!ctypes` will create descriptors in the structure type that allows accessing the nested fields directly, without the need to create the structure or union field. @@ -3010,7 +3162,7 @@ Arrays and pointers Abstract base class for arrays. The recommended way to create concrete array types is by multiplying any - :mod:`ctypes` data type with a non-negative integer. Alternatively, you can subclass + :mod:`!ctypes` data type with a non-negative integer. Alternatively, you can subclass this type and define :attr:`_length_` and :attr:`_type_` class variables. Array elements can be read and written using standard subscript and slice accesses; for slice reads, the resulting object is @@ -3036,10 +3188,10 @@ Arrays and pointers Create an array. Equivalent to ``type * length``, where *type* is a - :mod:`ctypes` data type and *length* an integer. + :mod:`!ctypes` data type and *length* an integer. - This function is :term:`soft deprecated` in favor of multiplication. - There are no plans to remove it. + .. soft-deprecated:: 3.14 + In favor of multiplication. .. class:: _Pointer diff --git a/Doc/library/curses.ascii.rst b/Doc/library/curses.ascii.rst index cb895664ff1..9ae82c14465 100644 --- a/Doc/library/curses.ascii.rst +++ b/Doc/library/curses.ascii.rst @@ -4,14 +4,11 @@ .. module:: curses.ascii :synopsis: Constants and set-membership functions for ASCII characters. -.. moduleauthor:: Eric S. Raymond -.. sectionauthor:: Eric S. Raymond - **Source code:** :source:`Lib/curses/ascii.py` -------------- -The :mod:`curses.ascii` module supplies name constants for ASCII characters and +The :mod:`!curses.ascii` module supplies name constants for ASCII characters and functions to test membership in various ASCII character classes. The constants supplied are names for control characters as follows: diff --git a/Doc/library/curses.panel.rst b/Doc/library/curses.panel.rst index 11fd841d381..5bc6b74b7f0 100644 --- a/Doc/library/curses.panel.rst +++ b/Doc/library/curses.panel.rst @@ -4,8 +4,6 @@ .. module:: curses.panel :synopsis: A panel stack extension that adds depth to curses windows. -.. sectionauthor:: A.M. Kuchling - -------------- Panels are windows with the added feature of depth, so they can be stacked on @@ -18,7 +16,7 @@ displayed. Panels can be added, moved up or down in the stack, and removed. Functions --------- -The module :mod:`curses.panel` defines the following functions: +The module :mod:`!curses.panel` defines the following functions: .. function:: bottom_panel() diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index fb84cf32246..0f1449873fc 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -4,16 +4,12 @@ .. module:: curses :synopsis: An interface to the curses library, providing portable terminal handling. - :platform: Unix - -.. sectionauthor:: Moshe Zadka -.. sectionauthor:: Eric Raymond **Source code:** :source:`Lib/curses` -------------- -The :mod:`curses` module provides an interface to the curses library, the +The :mod:`!curses` module provides an interface to the curses library, the de-facto standard for portable advanced terminal handling. While curses is most widely used in the Unix environment, versions are available @@ -23,6 +19,10 @@ Linux and the BSD variants of Unix. .. include:: ../includes/wasm-mobile-notavail.rst +.. include:: ../includes/optional-module.rst + +.. availability:: Unix. + .. note:: Whenever the documentation mentions a *character* it can be specified @@ -52,7 +52,7 @@ Linux and the BSD variants of Unix. Functions --------- -The module :mod:`curses` defines the following exception: +The module :mod:`!curses` defines the following exception: .. exception:: error @@ -65,7 +65,7 @@ The module :mod:`curses` defines the following exception: default to the current cursor location. Whenever *attr* is optional, it defaults to :const:`A_NORMAL`. -The module :mod:`curses` defines the following functions: +The module :mod:`!curses` defines the following functions: .. function:: assume_default_colors(fg, bg, /) @@ -579,7 +579,7 @@ The module :mod:`curses` defines the following functions: after :func:`initscr`. :func:`start_color` initializes eight basic colors (black, red, green, yellow, - blue, magenta, cyan, and white), and two global variables in the :mod:`curses` + blue, magenta, cyan, and white), and two global variables in the :mod:`!curses` module, :const:`COLORS` and :const:`COLOR_PAIRS`, containing the maximum number of colors and color-pairs the terminal can support. It also restores the colors on the terminal to the values they had when the terminal was just turned on. @@ -1019,7 +1019,7 @@ Window Objects .. method:: window.idlok(flag) - If *flag* is ``True``, :mod:`curses` will try and use hardware line + If *flag* is ``True``, :mod:`!curses` will try and use hardware line editing facilities. Otherwise, line insertion/deletion are disabled. @@ -1107,7 +1107,7 @@ Window Objects .. method:: window.keypad(flag) If *flag* is ``True``, escape sequences generated by some keys (keypad, function keys) - will be interpreted by :mod:`curses`. If *flag* is ``False``, escape sequences will be + will be interpreted by :mod:`!curses`. If *flag* is ``False``, escape sequences will be left as is in the input stream. @@ -1333,7 +1333,7 @@ Window Objects Constants --------- -The :mod:`curses` module defines the following data members: +The :mod:`!curses` module defines the following data members: .. data:: ERR @@ -1349,7 +1349,6 @@ The :mod:`curses` module defines the following data members: .. data:: version -.. data:: __version__ A bytes object representing the current version of the module. @@ -1823,22 +1822,19 @@ The following table lists the predefined colors: +-------------------------+----------------------------+ -:mod:`curses.textpad` --- Text input widget for curses programs -=============================================================== +:mod:`!curses.textpad` --- Text input widget for curses programs +================================================================ .. module:: curses.textpad :synopsis: Emacs-like input editing in a curses window. -.. moduleauthor:: Eric Raymond -.. sectionauthor:: Eric Raymond - -The :mod:`curses.textpad` module provides a :class:`Textbox` class that handles +The :mod:`!curses.textpad` module provides a :class:`Textbox` class that handles elementary text editing in a curses window, supporting a set of keybindings resembling those of Emacs (thus, also of Netscape Navigator, BBedit 6.x, FrameMaker, and many other programs). The module also provides a rectangle-drawing function useful for framing text boxes or for other purposes. -The module :mod:`curses.textpad` defines the following function: +The module :mod:`!curses.textpad` defines the following function: .. function:: rectangle(win, uly, ulx, lry, lrx) diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index ca432f2768a..0bce3e5b762 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -4,9 +4,6 @@ .. module:: dataclasses :synopsis: Generate special methods on user-defined classes. -.. moduleauthor:: Eric V. Smith -.. sectionauthor:: Eric V. Smith - **Source code:** :source:`Lib/dataclasses.py` -------------- @@ -317,7 +314,7 @@ Module contents :func:`!field`, then the class attribute for this field will be replaced by the specified *default* value. If *default* is not provided, then the class attribute will be deleted. The intent is - that after the :func:`@dataclass ` decorator runs, the class + that after the :deco:`dataclass` decorator runs, the class attributes will all contain the default values for the fields, just as if the default value itself were specified. For example, after:: @@ -333,6 +330,10 @@ Module contents :attr:`!C.t` will be ``20``, and the class attributes :attr:`!C.x` and :attr:`!C.y` will not be set. + .. versionchanged:: 3.15 + If *metadata* is ``None``, use an empty :class:`frozendict`, instead + of a :func:`~types.MappingProxyType` of an empty :class:`dict`. + .. class:: Field :class:`!Field` objects describe each defined field. These objects @@ -370,8 +371,8 @@ Module contents Converts the dataclass *obj* to a dict (by using the factory function *dict_factory*). Each dataclass is converted to a dict of its fields, as ``name: value`` pairs. dataclasses, dicts, - lists, and tuples are recursed into. Other objects are copied with - :func:`copy.deepcopy`. + frozendicts, lists, and tuples are recursed into. Other objects are copied + with :func:`copy.deepcopy`. Example of using :func:`!asdict` on nested dataclasses:: @@ -401,8 +402,8 @@ Module contents Converts the dataclass *obj* to a tuple (by using the factory function *tuple_factory*). Each dataclass is converted - to a tuple of its field values. dataclasses, dicts, lists, and - tuples are recursed into. Other objects are copied with + to a tuple of its field values. dataclasses, dicts, frozendicts, lists, + and tuples are recursed into. Other objects are copied with :func:`copy.deepcopy`. Continuing from the previous example:: @@ -427,7 +428,7 @@ Module contents :data:`typing.Any` is used for ``type``. The values of *init*, *repr*, *eq*, *order*, *unsafe_hash*, *frozen*, *match_args*, *kw_only*, *slots*, and *weakref_slot* have - the same meaning as they do in :func:`@dataclass `. + the same meaning as they do in :deco:`dataclass`. If *module* is defined, the :attr:`!__module__` attribute of the dataclass is set to that value. @@ -435,12 +436,12 @@ Module contents The *decorator* parameter is a callable that will be used to create the dataclass. It should take the class object as a first argument and the same keyword arguments - as :func:`@dataclass `. By default, the :func:`@dataclass ` + as :deco:`dataclass`. By default, the :deco:`dataclass` function is used. This function is not strictly required, because any Python mechanism for creating a new class with :attr:`~object.__annotations__` can - then apply the :func:`@dataclass ` function to convert that class to + then apply the :deco:`dataclass` function to convert that class to a dataclass. This function is provided as a convenience. For example:: @@ -569,7 +570,7 @@ Post-init processing def __post_init__(self): self.c = self.a + self.b -The :meth:`~object.__init__` method generated by :func:`@dataclass ` does not call base +The :meth:`~object.__init__` method generated by :deco:`dataclass` does not call base class :meth:`!__init__` methods. If the base class has an :meth:`!__init__` method that has to be called, it is common to call this method in a :meth:`__post_init__` method:: @@ -599,7 +600,7 @@ parameters to :meth:`!__post_init__`. Also see the warning about how Class variables --------------- -One of the few places where :func:`@dataclass ` actually inspects the type +One of the few places where :deco:`dataclass` actually inspects the type of a field is to determine if a field is a class variable as defined in :pep:`526`. It does this by checking if the type of the field is :data:`typing.ClassVar`. If a field is a ``ClassVar``, it is excluded @@ -612,7 +613,7 @@ module-level :func:`fields` function. Init-only variables ------------------- -Another place where :func:`@dataclass ` inspects a type annotation is to +Another place where :deco:`dataclass` inspects a type annotation is to determine if a field is an init-only variable. It does this by seeing if the type of a field is of type :class:`InitVar`. If a field is an :class:`InitVar`, it is considered a pseudo-field called an init-only @@ -646,7 +647,7 @@ Frozen instances ---------------- It is not possible to create truly immutable Python objects. However, -by passing ``frozen=True`` to the :func:`@dataclass ` decorator you can +by passing ``frozen=True`` to the :deco:`dataclass` decorator you can emulate immutability. In that case, dataclasses will add :meth:`~object.__setattr__` and :meth:`~object.__delattr__` methods to the class. These methods will raise a :exc:`FrozenInstanceError` when invoked. @@ -662,7 +663,7 @@ must use :meth:`!object.__setattr__`. Inheritance ----------- -When the dataclass is being created by the :func:`@dataclass ` decorator, +When the dataclass is being created by the :deco:`dataclass` decorator, it looks through all of the class's base classes in reverse MRO (that is, starting at :class:`object`) and, for each dataclass that it finds, adds the fields from that base class to an ordered mapping of fields. @@ -786,7 +787,7 @@ for :attr:`!x` when creating a class instance will share the same copy of :attr:`!x`. Because dataclasses just use normal Python class creation they also share this behavior. There is no general way for Data Classes to detect this condition. Instead, the -:func:`@dataclass ` decorator will raise a :exc:`ValueError` if it +:deco:`dataclass` decorator will raise a :exc:`ValueError` if it detects an unhashable default parameter. The assumption is that if a value is unhashable, it is mutable. This is a partial solution, but it does protect against many common errors. @@ -820,7 +821,7 @@ default value have the following special behaviors: :meth:`~object.__get__` or :meth:`!__set__` method is called rather than returning or overwriting the descriptor object. -* To determine whether a field contains a default value, :func:`@dataclass ` +* To determine whether a field contains a default value, :deco:`dataclass` will call the descriptor's :meth:`!__get__` method using its class access form: ``descriptor.__get__(obj=None, type=cls)``. If the descriptor returns a value in this case, it will be used as the diff --git a/Doc/library/datetime-inheritance.dot b/Doc/library/datetime-inheritance.dot new file mode 100644 index 00000000000..3c6b9b4beb7 --- /dev/null +++ b/Doc/library/datetime-inheritance.dot @@ -0,0 +1,31 @@ +// Used to generate datetime-inheritance.svg with Graphviz +// (https://graphviz.org/) for the datetime documentation. + +digraph { + comment="Generated with datetime-inheritance.dot" + graph [ + bgcolor="transparent" + fontnames="svg" + layout="dot" + ranksep=0.5 + nodesep=0.5 + splines=line + ] + node [ + fontname="Courier" + fontsize=14.0 + shape=box + style=rounded + margin="0.15,0.07" + ] + edge [ + arrowhead=none + ] + + object -> tzinfo + object -> timedelta + object -> time + object -> date + tzinfo -> timezone + date -> datetime +} diff --git a/Doc/library/datetime-inheritance.svg b/Doc/library/datetime-inheritance.svg new file mode 100644 index 00000000000..e6b1cf877a5 --- /dev/null +++ b/Doc/library/datetime-inheritance.svg @@ -0,0 +1,84 @@ + + + + + + +datetime class hierarchy + + +object + +object + + + +tzinfo + +tzinfo + + + +object->tzinfo + + + + +timedelta + +timedelta + + + +object->timedelta + + + + +time + +time + + + +object->time + + + + +date + +date + + + +object->date + + + + +timezone + +timezone + + + +tzinfo->timezone + + + + +datetime + +datetime + + + +date->datetime + + + + diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 7010f99c54d..f3c4ef91990 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -4,16 +4,10 @@ .. module:: datetime :synopsis: Basic date and time types. -.. moduleauthor:: Tim Peters -.. sectionauthor:: Tim Peters -.. sectionauthor:: A.M. Kuchling - **Source code:** :source:`Lib/datetime.py` -------------- -.. XXX what order should the types be discussed in? - The :mod:`!datetime` module supplies classes for manipulating dates and times. While date and time arithmetic is supported, the focus of the implementation is @@ -38,13 +32,14 @@ on efficient attribute extraction for output formatting and manipulation. Third-party library with expanded time zone and parsing support. Package :pypi:`DateType` - Third-party library that introduces distinct static types to e.g. allow - :term:`static type checkers ` + Third-party library that introduces distinct static types to for example, + allow :term:`static type checkers ` to differentiate between naive and aware datetimes. + .. _datetime-naive-aware: -Aware and Naive Objects +Aware and naive objects ----------------------- Date and time objects may be categorized as "aware" or "naive" depending on @@ -65,7 +60,7 @@ understand and to work with, at the cost of ignoring some aspects of reality. For applications requiring aware objects, :class:`.datetime` and :class:`.time` objects have an optional time zone information attribute, :attr:`!tzinfo`, that -can be set to an instance of a subclass of the abstract :class:`tzinfo` class. +can be set to an instance of a subclass of the abstract :class:`!tzinfo` class. These :class:`tzinfo` objects capture information about the offset from UTC time, the time zone name, and whether daylight saving time is in effect. @@ -77,6 +72,7 @@ detail is up to the application. The rules for time adjustment across the world are more political than rational, change frequently, and there is no standard suitable for every application aside from UTC. + Constants --------- @@ -93,13 +89,15 @@ The :mod:`!datetime` module exports the following constants: The largest year number allowed in a :class:`date` or :class:`.datetime` object. :const:`MAXYEAR` is 9999. + .. data:: UTC Alias for the UTC time zone singleton :attr:`datetime.timezone.utc`. .. versionadded:: 3.11 -Available Types + +Available types --------------- .. class:: date @@ -142,6 +140,7 @@ Available Types time adjustment (for example, to account for time zone and/or daylight saving time). + .. class:: timezone :noindex: @@ -150,19 +149,19 @@ Available Types .. versionadded:: 3.2 + Objects of these types are immutable. -Subclass relationships:: +Subclass relationships: - object - timedelta - tzinfo - timezone - time - date - datetime +.. figure:: datetime-inheritance.svg + :class: invert-in-dark-mode + :align: center + :alt: timedelta, tzinfo, time, and date inherit from object; timezone inherits + from tzinfo; and datetime inherits from date. -Common Properties + +Common properties ^^^^^^^^^^^^^^^^^ The :class:`date`, :class:`.datetime`, :class:`.time`, and :class:`timezone` types @@ -173,7 +172,8 @@ share these common features: dictionary keys. - Objects of these types support efficient pickling via the :mod:`pickle` module. -Determining if an Object is Aware or Naive + +Determining if an object is aware or naive ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Objects of the :class:`date` type are always naive. @@ -197,10 +197,11 @@ Otherwise, ``t`` is naive. The distinction between aware and naive doesn't apply to :class:`timedelta` objects. + .. _datetime-timedelta: -:class:`timedelta` Objects --------------------------- +:class:`!timedelta` objects +--------------------------- A :class:`timedelta` object represents a duration, the difference between two :class:`.datetime` or :class:`date` instances. @@ -229,8 +230,8 @@ A :class:`timedelta` object represents a duration, the difference between two *days*, *seconds* and *microseconds* are "merged" and normalized into those three resulting attributes:: - >>> from datetime import timedelta - >>> delta = timedelta( + >>> import datetime as dt + >>> delta = dt.timedelta( ... days=50, ... seconds=27, ... microseconds=10, @@ -243,6 +244,12 @@ A :class:`timedelta` object represents a duration, the difference between two >>> delta datetime.timedelta(days=64, seconds=29156, microseconds=10) + .. tip:: + ``import datetime as dt`` instead of ``import datetime`` or + ``from datetime import datetime`` to avoid confusion between the module + and the class. See `How I Import Python’s datetime Module + `__. + If any argument is a float and there are fractional microseconds, the fractional microseconds left over from all arguments are combined and their sum is rounded to the nearest microsecond using @@ -256,8 +263,8 @@ A :class:`timedelta` object represents a duration, the difference between two Note that normalization of negative values may be surprising at first. For example:: - >>> from datetime import timedelta - >>> d = timedelta(microseconds=-1) + >>> import datetime as dt + >>> d = dt.timedelta(microseconds=-1) >>> (d.days, d.seconds, d.microseconds) (-1, 86399, 999999) @@ -296,6 +303,7 @@ Class attributes: The smallest possible difference between non-equal :class:`timedelta` objects, ``timedelta(microseconds=1)``. + Note that, because of normalization, ``timedelta.max`` is greater than ``-timedelta.min``. ``-timedelta.max`` is not representable as a :class:`timedelta` object. @@ -319,13 +327,14 @@ Instance attributes (read-only): .. doctest:: - >>> from datetime import timedelta - >>> duration = timedelta(seconds=11235813) + >>> import datetime as dt + >>> duration = dt.timedelta(seconds=11235813) >>> duration.days, duration.seconds (130, 3813) >>> duration.total_seconds() 11235813.0 + .. attribute:: timedelta.microseconds Between 0 and 999,999 inclusive. @@ -333,8 +342,6 @@ Instance attributes (read-only): Supported operations: -.. XXX this table is too wide! - +--------------------------------+-----------------------------------------------+ | Operation | Result | +================================+===============================================+ @@ -396,7 +403,6 @@ Supported operations: | | call with canonical attribute values. | +--------------------------------+-----------------------------------------------+ - Notes: (1) @@ -432,9 +438,9 @@ objects (see below). .. versionchanged:: 3.2 Floor division and true division of a :class:`timedelta` object by another - :class:`timedelta` object are now supported, as are remainder operations and + :class:`!timedelta` object are now supported, as are remainder operations and the :func:`divmod` function. True division and multiplication of a - :class:`timedelta` object by a :class:`float` object are now supported. + :class:`!timedelta` object by a :class:`float` object are now supported. :class:`timedelta` objects support equality and order comparisons. @@ -447,23 +453,24 @@ Instance methods: Return the total number of seconds contained in the duration. Equivalent to ``td / timedelta(seconds=1)``. For interval units other than seconds, use the - division form directly (e.g. ``td / timedelta(microseconds=1)``). + division form directly (for example, ``td / timedelta(microseconds=1)``). Note that for very large time intervals (greater than 270 years on most platforms) this method will lose microsecond accuracy. .. versionadded:: 3.2 -Examples of usage: :class:`timedelta` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Examples of usage: :class:`!timedelta` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ An additional example of normalization:: >>> # Components of another_year add up to exactly 365 days - >>> from datetime import timedelta - >>> year = timedelta(days=365) - >>> another_year = timedelta(weeks=40, days=84, hours=23, - ... minutes=50, seconds=600) + >>> import datetime as dt + >>> year = dt.timedelta(days=365) + >>> another_year = dt.timedelta(weeks=40, days=84, hours=23, + ... minutes=50, seconds=600) >>> year == another_year True >>> year.total_seconds() @@ -471,8 +478,8 @@ An additional example of normalization:: Examples of :class:`timedelta` arithmetic:: - >>> from datetime import timedelta - >>> year = timedelta(days=365) + >>> import datetime as dt + >>> year = dt.timedelta(days=365) >>> ten_years = 10 * year >>> ten_years datetime.timedelta(days=3650) @@ -485,10 +492,11 @@ Examples of :class:`timedelta` arithmetic:: >>> three_years, three_years.days // 365 (datetime.timedelta(days=1095), 3) + .. _datetime-date: -:class:`date` Objects ---------------------- +:class:`!date` objects +---------------------- A :class:`date` object represents a date (year, month and day) in an idealized calendar, the current Gregorian calendar indefinitely extended in both @@ -517,9 +525,10 @@ Other constructors, all class methods: This is equivalent to ``date.fromtimestamp(time.time())``. + .. classmethod:: date.fromtimestamp(timestamp) - Return the local date corresponding to the POSIX timestamp, such as is + Return the local date corresponding to the POSIX *timestamp*, such as is returned by :func:`time.time`. This may raise :exc:`OverflowError`, if the timestamp is out @@ -535,10 +544,13 @@ Other constructors, all class methods: :c:func:`localtime` function. Raise :exc:`OSError` instead of :exc:`ValueError` on :c:func:`localtime` failure. + .. versionchanged:: 3.15 + Accepts any real number as *timestamp*, not only integer or float. + .. classmethod:: date.fromordinal(ordinal) - Return the date corresponding to the proleptic Gregorian ordinal, where + Return the date corresponding to the proleptic Gregorian *ordinal*, where January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1 <= ordinal <= @@ -559,25 +571,27 @@ Other constructors, all class methods: Examples:: - >>> from datetime import date - >>> date.fromisoformat('2019-12-04') + >>> import datetime as dt + >>> dt.date.fromisoformat('2019-12-04') datetime.date(2019, 12, 4) - >>> date.fromisoformat('20191204') + >>> dt.date.fromisoformat('20191204') datetime.date(2019, 12, 4) - >>> date.fromisoformat('2021-W01-1') + >>> dt.date.fromisoformat('2021-W01-1') datetime.date(2021, 1, 4) .. versionadded:: 3.7 .. versionchanged:: 3.11 Previously, this method only supported the format ``YYYY-MM-DD``. + .. classmethod:: date.fromisocalendar(year, week, day) Return a :class:`date` corresponding to the ISO calendar date specified by - year, week and day. This is the inverse of the function :meth:`date.isocalendar`. + *year*, *week* and *day*. This is the inverse of the function :meth:`date.isocalendar`. .. versionadded:: 3.8 + .. classmethod:: date.strptime(date_string, format) Return a :class:`.date` corresponding to *date_string*, parsed according to @@ -592,20 +606,19 @@ Other constructors, all class methods: .. note:: - If *format* specifies a day of month without a year a - :exc:`DeprecationWarning` is emitted. This is to avoid a quadrennial + If *format* specifies a day of month (``%d``) without a year, + :exc:`ValueError` is raised. This is to avoid a quadrennial leap year bug in code seeking to parse only a month and day as the default year used in absence of one in the format is not a leap year. - Such *format* values may raise an error as of Python 3.15. The - workaround is to always include a year in your *format*. If parsing + The workaround is to always include a year in your *format*. If parsing *date_string* values that do not have a year, explicitly add a year that is a leap year before parsing: .. doctest:: - >>> from datetime import date + >>> import datetime as dt >>> date_string = "02/29" - >>> when = date.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. + >>> when = dt.date.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. >>> when.strftime("%B %d") # doctest: +SKIP 'February 29' @@ -697,7 +710,7 @@ Notes: In other words, ``date1 < date2`` if and only if ``date1.toordinal() < date2.toordinal()``. - Order comparison between a :class:`!date` object that is not also a + Order comparison between a :class:`date` object that is not also a :class:`.datetime` instance and a :class:`!datetime` object raises :exc:`TypeError`. @@ -720,8 +733,8 @@ Instance methods: Example:: - >>> from datetime import date - >>> d = date(2002, 12, 31) + >>> import datetime as dt + >>> d = dt.date(2002, 12, 31) >>> d.replace(day=26) datetime.date(2002, 12, 26) @@ -779,23 +792,25 @@ Instance methods: For example, 2004 begins on a Thursday, so the first week of ISO year 2004 begins on Monday, 29 Dec 2003 and ends on Sunday, 4 Jan 2004:: - >>> from datetime import date - >>> date(2003, 12, 29).isocalendar() + >>> import datetime as dt + >>> dt.date(2003, 12, 29).isocalendar() datetime.IsoCalendarDate(year=2004, week=1, weekday=1) - >>> date(2004, 1, 4).isocalendar() + >>> dt.date(2004, 1, 4).isocalendar() datetime.IsoCalendarDate(year=2004, week=1, weekday=7) .. versionchanged:: 3.9 Result changed from a tuple to a :term:`named tuple`. + .. method:: date.isoformat() Return a string representing the date in ISO 8601 format, ``YYYY-MM-DD``:: - >>> from datetime import date - >>> date(2002, 12, 4).isoformat() + >>> import datetime as dt + >>> dt.date(2002, 12, 4).isoformat() '2002-12-04' + .. method:: date.__str__() For a date ``d``, ``str(d)`` is equivalent to ``d.isoformat()``. @@ -805,8 +820,8 @@ Instance methods: Return a string representing the date:: - >>> from datetime import date - >>> date(2002, 12, 4).ctime() + >>> import datetime as dt + >>> dt.date(2002, 12, 4).ctime() 'Wed Dec 4 00:00:00 2002' ``d.ctime()`` is equivalent to:: @@ -832,19 +847,20 @@ Instance methods: literals ` and when using :meth:`str.format`. See also :ref:`strftime-strptime-behavior` and :meth:`date.isoformat`. -Examples of Usage: :class:`date` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Examples of usage: :class:`!date` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Example of counting days to an event:: >>> import time - >>> from datetime import date - >>> today = date.today() + >>> import datetime as dt + >>> today = dt.date.today() >>> today datetime.date(2007, 12, 5) - >>> today == date.fromtimestamp(time.time()) + >>> today == dt.date.fromtimestamp(time.time()) True - >>> my_birthday = date(today.year, 6, 24) + >>> my_birthday = dt.date(today.year, 6, 24) >>> if my_birthday < today: ... my_birthday = my_birthday.replace(year=today.year + 1) ... @@ -858,8 +874,8 @@ More examples of working with :class:`date`: .. doctest:: - >>> from datetime import date - >>> d = date.fromordinal(730920) # 730920th day after 1. 1. 0001 + >>> import datetime as dt + >>> d = dt.date.fromordinal(730920) # 730920th day after 1. 1. 0001 >>> d datetime.date(2002, 3, 11) @@ -875,7 +891,7 @@ More examples of working with :class:`date`: >>> 'The {1} is {0:%d}, the {2} is {0:%B}.'.format(d, "day", "month") 'The day is 11, the month is March.' - >>> # Methods for to extracting 'components' under different calendars + >>> # Methods for extracting 'components' under different calendars >>> t = d.timetuple() >>> for i in t: # doctest: +SKIP ... print(i) @@ -902,7 +918,7 @@ More examples of working with :class:`date`: .. _datetime-datetime: -:class:`.datetime` Objects +:class:`!datetime` objects -------------------------- A :class:`.datetime` object is a single object containing all the information @@ -910,7 +926,7 @@ from a :class:`date` object and a :class:`.time` object. Like a :class:`date` object, :class:`.datetime` assumes the current Gregorian calendar extended in both directions; like a :class:`.time` object, -:class:`.datetime` assumes there are exactly 3600\*24 seconds in every day. +:class:`!datetime` assumes there are exactly 3600\*24 seconds in every day. Constructor: @@ -934,6 +950,7 @@ Constructor: .. versionchanged:: 3.6 Added the *fold* parameter. + Other constructors, all class methods: .. classmethod:: datetime.today() @@ -949,6 +966,7 @@ Other constructors, all class methods: This method is functionally equivalent to :meth:`now`, but without a ``tz`` parameter. + .. classmethod:: datetime.now(tz=None) Return the current local date and time. @@ -969,6 +987,7 @@ Other constructors, all class methods: Subsequent calls to :meth:`!datetime.now` may return the same instant depending on the precision of the underlying clock. + .. classmethod:: datetime.utcnow() Return the current UTC date and time, with :attr:`.tzinfo` ``None``. @@ -1020,6 +1039,10 @@ Other constructors, all class methods: .. versionchanged:: 3.6 :meth:`fromtimestamp` may return instances with :attr:`.fold` set to 1. + .. versionchanged:: 3.15 + Accepts any real number as *timestamp*, not only integer or float. + + .. classmethod:: datetime.utcfromtimestamp(timestamp) Return the UTC :class:`.datetime` corresponding to the POSIX timestamp, with @@ -1056,6 +1079,9 @@ Other constructors, all class methods: :c:func:`gmtime` function. Raise :exc:`OSError` instead of :exc:`ValueError` on :c:func:`gmtime` failure. + .. versionchanged:: 3.15 + Accepts any real number as *timestamp*, not only integer or float. + .. deprecated:: 3.12 Use :meth:`datetime.fromtimestamp` with :const:`UTC` instead. @@ -1076,7 +1102,7 @@ Other constructors, all class methods: are equal to the given :class:`.time` object's. If the *tzinfo* argument is provided, its value is used to set the :attr:`.tzinfo` attribute of the result, otherwise the :attr:`~.time.tzinfo` attribute of the *time* argument - is used. If the *date* argument is a :class:`.datetime` object, its time components + is used. If the *date* argument is a :class:`!datetime` object, its time components and :attr:`.tzinfo` attributes are ignored. For any :class:`.datetime` object ``d``, @@ -1102,24 +1128,24 @@ Other constructors, all class methods: Examples:: - >>> from datetime import datetime - >>> datetime.fromisoformat('2011-11-04') + >>> import datetime as dt + >>> dt.datetime.fromisoformat('2011-11-04') datetime.datetime(2011, 11, 4, 0, 0) - >>> datetime.fromisoformat('20111104') + >>> dt.datetime.fromisoformat('20111104') datetime.datetime(2011, 11, 4, 0, 0) - >>> datetime.fromisoformat('2011-11-04T00:05:23') + >>> dt.datetime.fromisoformat('2011-11-04T00:05:23') datetime.datetime(2011, 11, 4, 0, 5, 23) - >>> datetime.fromisoformat('2011-11-04T00:05:23Z') + >>> dt.datetime.fromisoformat('2011-11-04T00:05:23Z') datetime.datetime(2011, 11, 4, 0, 5, 23, tzinfo=datetime.timezone.utc) - >>> datetime.fromisoformat('20111104T000523') + >>> dt.datetime.fromisoformat('20111104T000523') datetime.datetime(2011, 11, 4, 0, 5, 23) - >>> datetime.fromisoformat('2011-W01-2T00:05:23.283') + >>> dt.datetime.fromisoformat('2011-W01-2T00:05:23.283') datetime.datetime(2011, 1, 4, 0, 5, 23, 283000) - >>> datetime.fromisoformat('2011-11-04 00:05:23.283') + >>> dt.datetime.fromisoformat('2011-11-04 00:05:23.283') datetime.datetime(2011, 11, 4, 0, 5, 23, 283000) - >>> datetime.fromisoformat('2011-11-04 00:05:23.283+00:00') + >>> dt.datetime.fromisoformat('2011-11-04 00:05:23.283+00:00') datetime.datetime(2011, 11, 4, 0, 5, 23, 283000, tzinfo=datetime.timezone.utc) - >>> datetime.fromisoformat('2011-11-04T00:05:23+04:00') # doctest: +NORMALIZE_WHITESPACE + >>> dt.datetime.fromisoformat('2011-11-04T00:05:23+04:00') # doctest: +NORMALIZE_WHITESPACE datetime.datetime(2011, 11, 4, 0, 5, 23, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))) @@ -1132,12 +1158,13 @@ Other constructors, all class methods: .. classmethod:: datetime.fromisocalendar(year, week, day) Return a :class:`.datetime` corresponding to the ISO calendar date specified - by year, week and day. The non-date components of the datetime are populated + by *year*, *week* and *day*. The non-date components of the datetime are populated with their normal default values. This is the inverse of the function :meth:`datetime.isocalendar`. .. versionadded:: 3.8 + .. classmethod:: datetime.strptime(date_string, format) Return a :class:`.datetime` corresponding to *date_string*, parsed according to @@ -1152,22 +1179,21 @@ Other constructors, all class methods: time tuple. See also :ref:`strftime-strptime-behavior` and :meth:`datetime.fromisoformat`. - .. versionchanged:: 3.13 + .. versionchanged:: 3.15 - If *format* specifies a day of month without a year a - :exc:`DeprecationWarning` is now emitted. This is to avoid a quadrennial + If *format* specifies a day of month (``%d``) without a year, + :exc:`ValueError` is raised. This is to avoid a quadrennial leap year bug in code seeking to parse only a month and day as the default year used in absence of one in the format is not a leap year. - Such *format* values may raise an error as of Python 3.15. The - workaround is to always include a year in your *format*. If parsing + The workaround is to always include a year in your *format*. If parsing *date_string* values that do not have a year, explicitly add a year that is a leap year before parsing: .. doctest:: - >>> from datetime import datetime + >>> import datetime as dt >>> date_string = "02/29" - >>> when = datetime.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. + >>> when = dt.datetime.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. >>> when.strftime("%B %d") # doctest: +SKIP 'February 29' @@ -1245,6 +1271,7 @@ Instance attributes (read-only): .. versionadded:: 3.6 + Supported operations: +---------------------------------------+--------------------------------+ @@ -1280,7 +1307,7 @@ Supported operations: datetime, and no time zone adjustments are done even if the input is aware. (3) - Subtraction of a :class:`.datetime` from a :class:`.datetime` is defined only if + Subtraction of a :class:`.datetime` from a :class:`!datetime` is defined only if both operands are naive, or if both are aware. If one is aware and the other is naive, :exc:`TypeError` is raised. @@ -1298,7 +1325,7 @@ Supported operations: :class:`.datetime` objects are equal if they represent the same date and time, taking into account the time zone. - Naive and aware :class:`!datetime` objects are never equal. + Naive and aware :class:`.datetime` objects are never equal. If both comparands are aware, and have the same :attr:`!tzinfo` attribute, the :attr:`!tzinfo` and :attr:`~.datetime.fold` attributes are ignored and @@ -1306,7 +1333,7 @@ Supported operations: If both comparands are aware and have different :attr:`~.datetime.tzinfo` attributes, the comparison acts as comparands were first converted to UTC datetimes except that the implementation never overflows. - :class:`!datetime` instances in a repeated interval are never equal to + :class:`.datetime` instances in a repeated interval are never equal to :class:`!datetime` instances in other time zone. (5) @@ -1335,6 +1362,7 @@ Supported operations: The default behavior can be changed by overriding the special comparison methods in subclasses. + Instance methods: .. method:: datetime.date() @@ -1490,11 +1518,13 @@ Instance methods: ``datetime.replace(tzinfo=timezone.utc)`` to make it aware, at which point you can use :meth:`.datetime.timetuple`. + .. method:: datetime.toordinal() Return the proleptic Gregorian ordinal of the date. The same as ``self.date().toordinal()``. + .. method:: datetime.timestamp() Return POSIX timestamp corresponding to the :class:`.datetime` @@ -1503,7 +1533,7 @@ Instance methods: Naive :class:`.datetime` instances are assumed to represent local time and this method relies on platform C functions to perform - the conversion. Since :class:`.datetime` supports a wider range of + the conversion. Since :class:`!datetime` supports a wider range of values than the platform C functions on many platforms, this method may raise :exc:`OverflowError` or :exc:`OSError` for times far in the past or far in the future. @@ -1513,16 +1543,6 @@ Instance methods: (dt - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() - .. versionadded:: 3.3 - - .. versionchanged:: 3.6 - The :meth:`timestamp` method uses the :attr:`.fold` attribute to - disambiguate the times during a repeated interval. - - .. versionchanged:: 3.6 - This method no longer relies on the platform C :c:func:`mktime` - function to perform conversions. - .. note:: There is no method to obtain the POSIX timestamp directly from a @@ -1537,6 +1557,17 @@ Instance methods: timestamp = (dt - datetime(1970, 1, 1)) / timedelta(seconds=1) + .. versionadded:: 3.3 + + .. versionchanged:: 3.6 + The :meth:`timestamp` method uses the :attr:`.fold` attribute to + disambiguate the times during a repeated interval. + + .. versionchanged:: 3.6 + This method no longer relies on the platform C :c:func:`mktime` + function to perform conversions. + + .. method:: datetime.weekday() Return the day of the week as an integer, where Monday is 0 and Sunday is 6. @@ -1572,24 +1603,24 @@ Instance methods: Examples:: - >>> from datetime import datetime, timezone - >>> datetime(2019, 5, 18, 15, 17, 8, 132263).isoformat() + >>> import datetime as dt + >>> dt.datetime(2019, 5, 18, 15, 17, 8, 132263).isoformat() '2019-05-18T15:17:08.132263' - >>> datetime(2019, 5, 18, 15, 17, tzinfo=timezone.utc).isoformat() + >>> dt.datetime(2019, 5, 18, 15, 17, tzinfo=dt.timezone.utc).isoformat() '2019-05-18T15:17:00+00:00' The optional argument *sep* (default ``'T'``) is a one-character separator, placed between the date and time portions of the result. For example:: - >>> from datetime import tzinfo, timedelta, datetime - >>> class TZ(tzinfo): + >>> import datetime as dt + >>> class TZ(dt.tzinfo): ... """A time zone with an arbitrary, constant -06:39 offset.""" - ... def utcoffset(self, dt): - ... return timedelta(hours=-6, minutes=-39) + ... def utcoffset(self, when): + ... return dt.timedelta(hours=-6, minutes=-39) ... - >>> datetime(2002, 12, 25, tzinfo=TZ()).isoformat(' ') + >>> dt.datetime(2002, 12, 25, tzinfo=TZ()).isoformat(' ') '2002-12-25 00:00:00-06:39' - >>> datetime(2009, 11, 27, microsecond=100, tzinfo=TZ()).isoformat() + >>> dt.datetime(2009, 11, 27, microsecond=100, tzinfo=TZ()).isoformat() '2009-11-27T00:00:00.000100-06:39' The optional argument *timespec* specifies the number of additional @@ -1613,11 +1644,11 @@ Instance methods: :exc:`ValueError` will be raised on an invalid *timespec* argument:: - >>> from datetime import datetime - >>> datetime.now().isoformat(timespec='minutes') # doctest: +SKIP + >>> import datetime as dt + >>> dt.datetime.now().isoformat(timespec='minutes') # doctest: +SKIP '2002-12-25T00:00' - >>> dt = datetime(2015, 1, 1, 12, 30, 59, 0) - >>> dt.isoformat(timespec='microseconds') + >>> my_datetime = dt.datetime(2015, 1, 1, 12, 30, 59, 0) + >>> my_datetime.isoformat(timespec='microseconds') '2015-01-01T12:30:59.000000' .. versionchanged:: 3.6 @@ -1634,8 +1665,8 @@ Instance methods: Return a string representing the date and time:: - >>> from datetime import datetime - >>> datetime(2002, 12, 4, 20, 30, 40).ctime() + >>> import datetime as dt + >>> dt.datetime(2002, 12, 4, 20, 30, 40).ctime() 'Wed Dec 4 20:30:40 2002' The output string will *not* include time zone information, regardless @@ -1665,34 +1696,34 @@ Instance methods: See also :ref:`strftime-strptime-behavior` and :meth:`datetime.isoformat`. -Examples of Usage: :class:`.datetime` +Examples of usage: :class:`!datetime` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Examples of working with :class:`.datetime` objects: .. doctest:: - >>> from datetime import datetime, date, time, timezone + >>> import datetime as dt >>> # Using datetime.combine() - >>> d = date(2005, 7, 14) - >>> t = time(12, 30) - >>> datetime.combine(d, t) + >>> d = dt.date(2005, 7, 14) + >>> t = dt.time(12, 30) + >>> dt.datetime.combine(d, t) datetime.datetime(2005, 7, 14, 12, 30) >>> # Using datetime.now() - >>> datetime.now() # doctest: +SKIP + >>> dt.datetime.now() # doctest: +SKIP datetime.datetime(2007, 12, 6, 16, 29, 43, 79043) # GMT +1 - >>> datetime.now(timezone.utc) # doctest: +SKIP + >>> dt.datetime.now(dt.timezone.utc) # doctest: +SKIP datetime.datetime(2007, 12, 6, 15, 29, 43, 79060, tzinfo=datetime.timezone.utc) >>> # Using datetime.strptime() - >>> dt = datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M") - >>> dt + >>> my_datetime = dt.datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M") + >>> my_datetime datetime.datetime(2006, 11, 21, 16, 30) >>> # Using datetime.timetuple() to get tuple of all attributes - >>> tt = dt.timetuple() + >>> tt = my_datetime.timetuple() >>> for it in tt: # doctest: +SKIP ... print(it) ... @@ -1707,7 +1738,7 @@ Examples of working with :class:`.datetime` objects: -1 # dst - method tzinfo.dst() returned None >>> # Date in ISO format - >>> ic = dt.isocalendar() + >>> ic = my_datetime.isocalendar() >>> for it in ic: # doctest: +SKIP ... print(it) ... @@ -1716,55 +1747,55 @@ Examples of working with :class:`.datetime` objects: 2 # ISO weekday >>> # Formatting a datetime - >>> dt.strftime("%A, %d. %B %Y %I:%M%p") + >>> my_datetime.strftime("%A, %d. %B %Y %I:%M%p") 'Tuesday, 21. November 2006 04:30PM' - >>> 'The {1} is {0:%d}, the {2} is {0:%B}, the {3} is {0:%I:%M%p}.'.format(dt, "day", "month", "time") + >>> 'The {1} is {0:%d}, the {2} is {0:%B}, the {3} is {0:%I:%M%p}.'.format(my_datetime, "day", "month", "time") 'The day is 21, the month is November, the time is 04:30PM.' The example below defines a :class:`tzinfo` subclass capturing time zone information for Kabul, Afghanistan, which used +4 UTC until 1945 and then +4:30 UTC thereafter:: - from datetime import timedelta, datetime, tzinfo, timezone + import datetime as dt - class KabulTz(tzinfo): + class KabulTz(dt.tzinfo): # Kabul used +4 until 1945, when they moved to +4:30 - UTC_MOVE_DATE = datetime(1944, 12, 31, 20, tzinfo=timezone.utc) + UTC_MOVE_DATE = dt.datetime(1944, 12, 31, 20, tzinfo=dt.timezone.utc) - def utcoffset(self, dt): - if dt.year < 1945: - return timedelta(hours=4) - elif (1945, 1, 1, 0, 0) <= dt.timetuple()[:5] < (1945, 1, 1, 0, 30): + def utcoffset(self, when): + if when.year < 1945: + return dt.timedelta(hours=4) + elif (1945, 1, 1, 0, 0) <= when.timetuple()[:5] < (1945, 1, 1, 0, 30): # An ambiguous ("imaginary") half-hour range representing # a 'fold' in time due to the shift from +4 to +4:30. - # If dt falls in the imaginary range, use fold to decide how - # to resolve. See PEP495. - return timedelta(hours=4, minutes=(30 if dt.fold else 0)) + # If when falls in the imaginary range, use fold to decide how + # to resolve. See PEP 495. + return dt.timedelta(hours=4, minutes=(30 if when.fold else 0)) else: - return timedelta(hours=4, minutes=30) + return dt.timedelta(hours=4, minutes=30) - def fromutc(self, dt): + def fromutc(self, when): # Follow same validations as in datetime.tzinfo - if not isinstance(dt, datetime): + if not isinstance(when, dt.datetime): raise TypeError("fromutc() requires a datetime argument") - if dt.tzinfo is not self: - raise ValueError("dt.tzinfo is not self") + if when.tzinfo is not self: + raise ValueError("when.tzinfo is not self") # A custom implementation is required for fromutc as # the input to this function is a datetime with utc values # but with a tzinfo set to self. # See datetime.astimezone or fromtimestamp. - if dt.replace(tzinfo=timezone.utc) >= self.UTC_MOVE_DATE: - return dt + timedelta(hours=4, minutes=30) + if when.replace(tzinfo=dt.timezone.utc) >= self.UTC_MOVE_DATE: + return when + dt.timedelta(hours=4, minutes=30) else: - return dt + timedelta(hours=4) + return when + dt.timedelta(hours=4) - def dst(self, dt): + def dst(self, when): # Kabul does not observe daylight saving time. - return timedelta(0) + return dt.timedelta(0) - def tzname(self, dt): - if dt >= self.UTC_MOVE_DATE: + def tzname(self, when): + if when >= self.UTC_MOVE_DATE: return "+04:30" return "+04" @@ -1773,17 +1804,17 @@ Usage of ``KabulTz`` from above:: >>> tz1 = KabulTz() >>> # Datetime before the change - >>> dt1 = datetime(1900, 11, 21, 16, 30, tzinfo=tz1) + >>> dt1 = dt.datetime(1900, 11, 21, 16, 30, tzinfo=tz1) >>> print(dt1.utcoffset()) 4:00:00 >>> # Datetime after the change - >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=tz1) + >>> dt2 = dt.datetime(2006, 6, 14, 13, 0, tzinfo=tz1) >>> print(dt2.utcoffset()) 4:30:00 >>> # Convert datetime to another time zone - >>> dt3 = dt2.astimezone(timezone.utc) + >>> dt3 = dt2.astimezone(dt.timezone.utc) >>> dt3 datetime.datetime(2006, 6, 14, 8, 30, tzinfo=datetime.timezone.utc) >>> dt2 @@ -1791,9 +1822,10 @@ Usage of ``KabulTz`` from above:: >>> dt2 == dt3 True + .. _datetime-time: -:class:`.time` Objects +:class:`!time` objects ---------------------- A :class:`.time` object represents a (local) time of day, independent of any particular @@ -1814,6 +1846,7 @@ day, and subject to adjustment via a :class:`tzinfo` object. If an argument outside those ranges is given, :exc:`ValueError` is raised. All default to 0 except *tzinfo*, which defaults to ``None``. + Class attributes: @@ -1872,6 +1905,7 @@ Instance attributes (read-only): .. versionadded:: 3.6 + :class:`.time` objects support equality and order comparisons, where ``a`` is considered less than ``b`` when ``a`` precedes ``b`` in time. @@ -1894,8 +1928,8 @@ In Boolean contexts, a :class:`.time` object is always considered to be true. .. versionchanged:: 3.5 Before Python 3.5, a :class:`.time` object was considered to be false if it represented midnight in UTC. This behavior was considered obscure and - error-prone and has been removed in Python 3.5. See :issue:`13936` for full - details. + error-prone and has been removed in Python 3.5. See :issue:`13936` for more + information. Other constructors: @@ -1916,22 +1950,22 @@ Other constructors: .. doctest:: - >>> from datetime import time - >>> time.fromisoformat('04:23:01') + >>> import datetime as dt + >>> dt.time.fromisoformat('04:23:01') datetime.time(4, 23, 1) - >>> time.fromisoformat('T04:23:01') + >>> dt.time.fromisoformat('T04:23:01') datetime.time(4, 23, 1) - >>> time.fromisoformat('T042301') + >>> dt.time.fromisoformat('T042301') datetime.time(4, 23, 1) - >>> time.fromisoformat('04:23:01.000384') + >>> dt.time.fromisoformat('04:23:01.000384') datetime.time(4, 23, 1, 384) - >>> time.fromisoformat('04:23:01,000384') + >>> dt.time.fromisoformat('04:23:01,000384') datetime.time(4, 23, 1, 384) - >>> time.fromisoformat('04:23:01+04:00') + >>> dt.time.fromisoformat('04:23:01+04:00') datetime.time(4, 23, 1, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))) - >>> time.fromisoformat('04:23:01Z') + >>> dt.time.fromisoformat('04:23:01Z') datetime.time(4, 23, 1, tzinfo=datetime.timezone.utc) - >>> time.fromisoformat('04:23:01+00:00') + >>> dt.time.fromisoformat('04:23:01+00:00') datetime.time(4, 23, 1, tzinfo=datetime.timezone.utc) @@ -1940,6 +1974,7 @@ Other constructors: Previously, this method only supported formats that could be emitted by :meth:`time.isoformat`. + .. classmethod:: time.strptime(date_string, format) Return a :class:`.time` corresponding to *date_string*, parsed according to @@ -1964,7 +1999,7 @@ Instance methods: Return a new :class:`.time` with the same values, but with specified parameters updated. Note that ``tzinfo=None`` can be specified to create a - naive :class:`.time` from an aware :class:`.time`, without conversion of the + naive :class:`!time` from an aware :class:`!time`, without conversion of the time data. :class:`.time` objects are also supported by generic function @@ -2005,13 +2040,13 @@ Instance methods: Example:: - >>> from datetime import time - >>> time(hour=12, minute=34, second=56, microsecond=123456).isoformat(timespec='minutes') + >>> import datetime as dt + >>> dt.time(hour=12, minute=34, second=56, microsecond=123456).isoformat(timespec='minutes') '12:34' - >>> dt = time(hour=12, minute=34, second=56, microsecond=0) - >>> dt.isoformat(timespec='microseconds') + >>> my_time = dt.time(hour=12, minute=34, second=56, microsecond=0) + >>> my_time.isoformat(timespec='microseconds') '12:34:56.000000' - >>> dt.isoformat(timespec='auto') + >>> my_time.isoformat(timespec='auto') '12:34:56' .. versionchanged:: 3.6 @@ -2056,29 +2091,31 @@ Instance methods: .. versionchanged:: 3.7 The DST offset is not restricted to a whole number of minutes. + .. method:: time.tzname() If :attr:`.tzinfo` is ``None``, returns ``None``, else returns ``self.tzinfo.tzname(None)``, or raises an exception if the latter doesn't return ``None`` or a string object. -Examples of Usage: :class:`.time` + +Examples of usage: :class:`!time` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Examples of working with a :class:`.time` object:: - >>> from datetime import time, tzinfo, timedelta - >>> class TZ1(tzinfo): - ... def utcoffset(self, dt): - ... return timedelta(hours=1) - ... def dst(self, dt): - ... return timedelta(0) - ... def tzname(self,dt): + >>> import datetime as dt + >>> class TZ1(dt.tzinfo): + ... def utcoffset(self, when): + ... return dt.timedelta(hours=1) + ... def dst(self, when): + ... return dt.timedelta(0) + ... def tzname(self, when): ... return "+01:00" ... def __repr__(self): ... return f"{self.__class__.__name__}()" ... - >>> t = time(12, 10, 30, tzinfo=TZ1()) + >>> t = dt.time(12, 10, 30, tzinfo=TZ1()) >>> t datetime.time(12, 10, 30, tzinfo=TZ1()) >>> t.isoformat() @@ -2095,25 +2132,25 @@ Examples of working with a :class:`.time` object:: .. _datetime-tzinfo: -:class:`tzinfo` Objects ------------------------ +:class:`!tzinfo` objects +------------------------ .. class:: tzinfo() - This is an abstract base class, meaning that this class should not be + This is an :term:`abstract base class`, meaning that this class should not be instantiated directly. Define a subclass of :class:`tzinfo` to capture information about a particular time zone. An instance of (a concrete subclass of) :class:`tzinfo` can be passed to the constructors for :class:`.datetime` and :class:`.time` objects. The latter objects - view their attributes as being in local time, and the :class:`tzinfo` object + view their attributes as being in local time, and the :class:`!tzinfo` object supports methods revealing offset of local time from UTC, the name of the time zone, and DST offset, all relative to a date or time object passed to them. You need to derive a concrete subclass, and (at least) supply implementations of the standard :class:`tzinfo` methods needed by the :class:`.datetime` methods you use. The :mod:`!datetime` module provides - :class:`timezone`, a simple concrete subclass of :class:`tzinfo` which can + :class:`timezone`, a simple concrete subclass of :class:`!tzinfo` which can represent time zones with fixed offset from UTC such as UTC itself or North American EST and EDT. @@ -2176,31 +2213,35 @@ Examples of working with a :class:`.time` object:: ``tz.utcoffset(dt) - tz.dst(dt)`` must return the same result for every :class:`.datetime` *dt* with ``dt.tzinfo == - tz``. For sane :class:`tzinfo` subclasses, this expression yields the time + tz``. For sane :class:`!tzinfo` subclasses, this expression yields the time zone's "standard offset", which should not depend on the date or the time, but only on geographic location. The implementation of :meth:`datetime.astimezone` relies on this, but cannot detect violations; it's the programmer's - responsibility to ensure it. If a :class:`tzinfo` subclass cannot guarantee + responsibility to ensure it. If a :class:`!tzinfo` subclass cannot guarantee this, it may be able to override the default implementation of :meth:`tzinfo.fromutc` to work correctly with :meth:`~.datetime.astimezone` regardless. Most implementations of :meth:`dst` will probably look like one of these two:: - def dst(self, dt): + import datetime as dt + + def dst(self, when): # a fixed-offset class: doesn't account for DST - return timedelta(0) + return dt.timedelta(0) or:: - def dst(self, dt): + import datetime as dt + + def dst(self, when): # Code to set dston and dstoff to the time zone's DST - # transition times based on the input dt.year, and expressed + # transition times based on the input when.year, and expressed # in standard local time. - if dston <= dt.replace(tzinfo=None) < dstoff: - return timedelta(hours=1) + if dston <= when.replace(tzinfo=None) < dstoff: + return dt.timedelta(hours=1) else: - return timedelta(0) + return dt.timedelta(0) The default implementation of :meth:`dst` raises :exc:`NotImplementedError`. @@ -2217,17 +2258,17 @@ Examples of working with a :class:`.time` object:: valid replies. Return ``None`` if a string name isn't known. Note that this is a method rather than a fixed string primarily because some :class:`tzinfo` subclasses will wish to return different names depending on the specific value - of *dt* passed, especially if the :class:`tzinfo` class is accounting for + of *dt* passed, especially if the :class:`!tzinfo` class is accounting for daylight time. The default implementation of :meth:`tzname` raises :exc:`NotImplementedError`. These methods are called by a :class:`.datetime` or :class:`.time` object, in -response to their methods of the same names. A :class:`.datetime` object passes -itself as the argument, and a :class:`.time` object passes ``None`` as the +response to their methods of the same names. A :class:`!datetime` object passes +itself as the argument, and a :class:`!time` object passes ``None`` as the argument. A :class:`tzinfo` subclass's methods should therefore be prepared to -accept a *dt* argument of ``None``, or of class :class:`.datetime`. +accept a *dt* argument of ``None``, or of class :class:`!datetime`. When ``None`` is passed, it's up to the class designer to decide the best response. For example, returning ``None`` is appropriate if the class wishes to @@ -2235,10 +2276,10 @@ say that time objects don't participate in the :class:`tzinfo` protocols. It may be more useful for ``utcoffset(None)`` to return the standard UTC offset, as there is no other convention for discovering the standard offset. -When a :class:`.datetime` object is passed in response to a :class:`.datetime` +When a :class:`.datetime` object is passed in response to a :class:`!datetime` method, ``dt.tzinfo`` is the same object as *self*. :class:`tzinfo` methods can -rely on this, unless user code calls :class:`tzinfo` methods directly. The -intent is that the :class:`tzinfo` methods interpret *dt* as being in local +rely on this, unless user code calls :class:`!tzinfo` methods directly. The +intent is that the :class:`!tzinfo` methods interpret *dt* as being in local time, and not need worry about objects in other time zones. There is one more :class:`tzinfo` method that a subclass may wish to override: @@ -2266,20 +2307,22 @@ There is one more :class:`tzinfo` method that a subclass may wish to override: Skipping code for error cases, the default :meth:`fromutc` implementation acts like:: - def fromutc(self, dt): - # raise ValueError error if dt.tzinfo is not self - dtoff = dt.utcoffset() - dtdst = dt.dst() + import datetime as dt + + def fromutc(self, when): + # raise ValueError error if when.tzinfo is not self + dtoff = when.utcoffset() + dtdst = when.dst() # raise ValueError if dtoff is None or dtdst is None delta = dtoff - dtdst # this is self's standard offset if delta: - dt += delta # convert to standard local time - dtdst = dt.dst() + when += delta # convert to standard local time + dtdst = when.dst() # raise ValueError if dtdst is None if dtdst: - return dt + dtdst + return when + dtdst else: - return dt + return when In the following :download:`tzinfo_examples.py <../includes/tzinfo_examples.py>` file there are some examples of @@ -2306,9 +2349,9 @@ When DST starts (the "start" line), the local wall clock leaps from 1:59 to ``astimezone(Eastern)`` won't deliver a result with ``hour == 2`` on the day DST begins. For example, at the Spring forward transition of 2016, we get:: - >>> from datetime import datetime, timezone + >>> import datetime as dt >>> from tzinfo_examples import HOUR, Eastern - >>> u0 = datetime(2016, 3, 13, 5, tzinfo=timezone.utc) + >>> u0 = dt.datetime(2016, 3, 13, 5, tzinfo=dt.timezone.utc) >>> for i in range(4): ... u = u0 + i*HOUR ... t = u.astimezone(Eastern) @@ -2331,7 +2374,9 @@ form 5:MM and 6:MM both map to 1:MM when converted to Eastern, but earlier times have the :attr:`~.datetime.fold` attribute set to 0 and the later times have it set to 1. For example, at the Fall back transition of 2016, we get:: - >>> u0 = datetime(2016, 11, 6, 4, tzinfo=timezone.utc) + >>> import datetime as dt + >>> from tzinfo_examples import HOUR, Eastern + >>> u0 = dt.datetime(2016, 11, 6, 4, tzinfo=dt.timezone.utc) >>> for i in range(4): ... u = u0 + i*HOUR ... t = u.astimezone(Eastern) @@ -2348,7 +2393,7 @@ Note that the :class:`.datetime` instances that differ only by the value of the Applications that can't bear wall-time ambiguities should explicitly check the value of the :attr:`~.datetime.fold` attribute or avoid using hybrid :class:`tzinfo` subclasses; there are no ambiguities when using :class:`timezone`, -or any other fixed-offset :class:`tzinfo` subclass (such as a class representing +or any other fixed-offset :class:`!tzinfo` subclass (such as a class representing only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. seealso:: @@ -2371,8 +2416,8 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. _datetime-timezone: -:class:`timezone` Objects -------------------------- +:class:`!timezone` objects +-------------------------- The :class:`timezone` class is a subclass of :class:`tzinfo`, each instance of which represents a time zone defined by a fixed offset from @@ -2410,6 +2455,7 @@ where historical changes have been made to civil time. .. versionchanged:: 3.7 The UTC offset is not restricted to a whole number of minutes. + .. method:: timezone.tzname(dt) Return the fixed value specified when the :class:`timezone` instance @@ -2430,11 +2476,13 @@ where historical changes have been made to civil time. Always returns ``None``. + .. method:: timezone.fromutc(dt) Return ``dt + offset``. The *dt* argument must be an aware :class:`.datetime` instance, with ``tzinfo`` set to ``self``. + Class attributes: .. attribute:: timezone.utc @@ -2447,8 +2495,8 @@ Class attributes: .. _strftime-strptime-behavior: -:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Behavior --------------------------------------------------------------------- +:meth:`!strftime` and :meth:`!strptime` behavior +------------------------------------------------ :class:`date`, :class:`.datetime`, and :class:`.time` objects all support a ``strftime(format)`` method, to create a string representing the time under the @@ -2474,90 +2522,120 @@ versus :meth:`~.datetime.strptime`: .. _format-codes: -:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Format Codes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:meth:`!strftime` and :meth:`!strptime` format codes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ These methods accept format codes that can be used to parse and format dates:: - >>> datetime.strptime('31/01/22 23:59:59.999999', - ... '%d/%m/%y %H:%M:%S.%f') + >>> import datetime as dt + >>> dt.datetime.strptime('31/01/22 23:59:59.999999', + ... '%d/%m/%y %H:%M:%S.%f') datetime.datetime(2022, 1, 31, 23, 59, 59, 999999) >>> _.strftime('%a %d %b %Y, %I:%M%p') 'Mon 31 Jan 2022, 11:59PM' -The following is a list of all the format codes that the 1989 C standard -requires, and these work on all platforms with a standard C implementation. +The following is a list of all the format codes that the 2011 C standard +requires, and these work on all supported platforms. +-----------+--------------------------------+------------------------+-------+ -| Directive | Meaning | Example | Notes | +| Directive | Meaning | Example | Notes | +| | | | | +===========+================================+========================+=======+ -| ``%a`` | Weekday as locale's || Sun, Mon, ..., Sat | \(1) | +| ``%a`` | Weekday as locale's || Sun, Mon, ..., Sat | \(1) | | | abbreviated name. | (en_US); | | | | || So, Mo, ..., Sa | | | | | (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%A`` | Weekday as locale's full name. || Sunday, Monday, ..., | \(1) | +| ``%A`` | Weekday as locale's full name. || Sunday, Monday, ..., | \(1) | | | | Saturday (en_US); | | | | || Sonntag, Montag, ..., | | | | | Samstag (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%w`` | Weekday as a decimal number, | 0, 1, ..., 6 | | -| | where 0 is Sunday and 6 is | | | -| | Saturday. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%d`` | Day of the month as a | 01, 02, ..., 31 | \(9) | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%b`` | Month as locale's abbreviated || Jan, Feb, ..., Dec | \(1) | +| ``%b`` | Month as locale's abbreviated || Jan, Feb, ..., Dec | \(1) | | | name. | (en_US); | | | | || Jan, Feb, ..., Dez | | | | | (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%B`` | Month as locale's full name. || January, February, | \(1) | +| ``%B`` | Month as locale's full name. || January, February, | \(1) | | | | ..., December (en_US);| | | | || Januar, Februar, ..., | | | | | Dezember (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | \(9) | +| ``%c`` | Locale's appropriate date and || Tue Aug 16 21:30:00 | \(1) | +| | time representation. | 1988 (en_US); | | +| | || Di 16 Aug 21:30:00 | | +| | | 1988 (de_DE) | | ++-----------+--------------------------------+------------------------+-------+ +| ``%C`` | The year divided by 100 and | 01, 02, ..., 99 | \(0) | +| | truncated to an integer as a | | | +| | zero-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%d`` | Day of the month as a | 01, 02, ..., 31 | \(9), | +| | zero-padded decimal number. | | \(10) | ++-----------+--------------------------------+------------------------+-------+ +| ``%D`` | Equivalent to ``%m/%d/%y``. | 11/28/25 | \(9) | +| | | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%e`` | The day of the month as a | ␣1, ␣2, ..., 31 | \(10) | +| | space-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%F`` | Equivalent to ``%Y-%m-%d``, | 2025-10-11, | | +| | the ISO 8601 format. | 1001-12-30 | | ++-----------+--------------------------------+------------------------+-------+ +| ``%g`` | Last 2 digits of ISO 8601 year | 00, 01, ..., 99 | \(0) | +| | representing the year that | | | +| | contains the greater part of | | | +| | the ISO week (``%V``). | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%G`` | ISO 8601 year with century | 0001, 0002, ..., 2013, | \(8) | +| | representing the year that | 2014, ..., 9998, 9999 | | +| | contains the greater part of | | | +| | the ISO week (``%V``). | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%h`` | Equivalent to ``%b``. | See ``%b``. | \(0) | ++-----------+--------------------------------+------------------------+-------+ +| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | \(9) | +| | zero-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | \(9) | +| | zero-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%j`` | Day of the year as a | 001, 002, ..., 366 | \(9) | +| | zero-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | \(9) | | | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%y`` | Year without century as a | 00, 01, ..., 99 | \(9) | -| | zero-padded decimal number. | | | +| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | \(9) | +| | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%Y`` | Year with century as a decimal | 0001, 0002, ..., 2013, | \(2) | -| | number. | 2014, ..., 9998, 9999 | | +| ``%n`` | The newline character | ``\n`` | | +| | (``'\n'``). For | | | +| | :meth:`!strptime`, zero or | | | +| | more whitespace. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | \(9) | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | \(9) | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%p`` | Locale's equivalent of either || AM, PM (en_US); | \(1), | +| ``%p`` | Locale's equivalent of either || AM, PM (en_US); | \(1), | | | AM or PM. || am, pm (de_DE) | \(3) | +-----------+--------------------------------+------------------------+-------+ -| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | \(9) | -| | decimal number. | | | +| ``%r`` | Locale's 12-hour clock time. | 12:00:00 AM | \(1), | +| | | | \(0) | +-----------+--------------------------------+------------------------+-------+ -| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(4), | +| ``%R`` | Equivalent to ``%H:%M``. | 10:01 | | ++-----------+--------------------------------+------------------------+-------+ +| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(4), | | | decimal number. | | \(9) | +-----------+--------------------------------+------------------------+-------+ -| ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(5) | -| | number, zero-padded to 6 | 999999 | | -| | digits. | | | +| ``%t`` | The tab character (``'\t'``). | ``\t`` | | +| | For :meth:`!strptime`, | | | +| | zero or more whitespace. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%z`` | UTC offset in the form | (empty), +0000, | \(6) | -| | ``±HHMM[SS[.ffffff]]`` (empty | -0400, +1030, | | -| | string if the object is | +063415, | | -| | naive). | -030712.345216 | | +| ``%T`` | ISO 8601 time format, | 10:01:59 | | +| | equivalent to ``%H:%M:%S``. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%Z`` | Time zone name (empty string | (empty), UTC, GMT | \(6) | -| | if the object is naive). | | | +| ``%u`` | ISO 8601 weekday as a decimal | 1, 2, ..., 7 | | +| | number where 1 is Monday. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%j`` | Day of the year as a | 001, 002, ..., 366 | \(9) | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7), | +| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7), | | | (Sunday as the first day of | | \(9) | | | the week) as a zero-padded | | | | | decimal number. All days in a | | | @@ -2565,7 +2643,17 @@ requires, and these work on all platforms with a standard C implementation. | | Sunday are considered to be in | | | | | week 0. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7), | +| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8), | +| | number with Monday as | | \(9) | +| | the first day of the week. | | | +| | Week 01 is the week containing | | | +| | Jan 4. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%w`` | Weekday as a decimal number, | 0, 1, ..., 6 | | +| | where 0 is Sunday and 6 is | | | +| | Saturday. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7), | | | (Monday as the first day of | | \(9) | | | the week) as a zero-padded | | | | | decimal number. All days in a | | | @@ -2573,40 +2661,43 @@ requires, and these work on all platforms with a standard C implementation. | | Monday are considered to be in | | | | | week 0. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%c`` | Locale's appropriate date and || Tue Aug 16 21:30:00 | \(1) | -| | time representation. | 1988 (en_US); | | -| | || Di 16 Aug 21:30:00 | | -| | | 1988 (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%x`` | Locale's appropriate date || 08/16/88 (None); | \(1) | +| ``%x`` | Locale's appropriate date || 08/16/88 (None); | \(1) | | | representation. || 08/16/1988 (en_US); | | | | || 16.08.1988 (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%X`` | Locale's appropriate time || 21:30:00 (en_US); | \(1) | +| ``%X`` | Locale's appropriate time || 21:30:00 (en_US); | \(1) | | | representation. || 21:30:00 (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%%`` | A literal ``'%'`` character. | % | | +| ``%y`` | Year without century as a | 00, 01, ..., 99 | \(9) | +| | zero-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%Y`` | Year with century as a decimal | 0001, 0002, ..., 2013, | \(2) | +| | number. | 2014, ..., 9998, 9999 | | ++-----------+--------------------------------+------------------------+-------+ +| ``%z`` | UTC offset in the form | (empty), +0000, | \(6) | +| | ``±HHMM[SS[.ffffff]]`` (empty | -0400, +1030, | | +| | string if the object is | +063415, | | +| | naive). | -030712.345216 | | ++-----------+--------------------------------+------------------------+-------+ +| ``%Z`` | Time zone name (empty string | (empty), UTC, GMT | \(6) | +| | if the object is naive). | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%%`` | A literal ``'%'`` character. | % | | +-----------+--------------------------------+------------------------+-------+ -Several additional directives not required by the C89 standard are included for -convenience. These parameters all correspond to ISO 8601 date values. +The ISO 8601 year and ISO 8601 week directives are not interchangeable +with the year and week number directives above. Calling :meth:`~.datetime.strptime` with +incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. + +Several additional directives not required by the C11 standard are included for +convenience. +-----------+--------------------------------+------------------------+-------+ | Directive | Meaning | Example | Notes | +===========+================================+========================+=======+ -| ``%G`` | ISO 8601 year with century | 0001, 0002, ..., 2013, | \(8) | -| | representing the year that | 2014, ..., 9998, 9999 | | -| | contains the greater part of | | | -| | the ISO week (``%V``). | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%u`` | ISO 8601 weekday as a decimal | 1, 2, ..., 7 | | -| | number where 1 is Monday. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8), | -| | number with Monday as | | \(9) | -| | the first day of the week. | | | -| | Week 01 is the week containing | | | -| | Jan 4. | | | +| ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(5) | +| | number, zero-padded to 6 | 999999 | | +| | digits. | | | +-----------+--------------------------------+------------------------+-------+ | ``%:z`` | UTC offset in the form | (empty), +00:00, | \(6) | | | ``±HH:MM[:SS[.ffffff]]`` | -04:00, +10:30, | | @@ -2614,11 +2705,6 @@ convenience. These parameters all correspond to ISO 8601 date values. | | naive). | -03:07:12.345216 | | +-----------+--------------------------------+------------------------+-------+ -These may not be available on all platforms when used with the :meth:`~.datetime.strftime` -method. The ISO 8601 year and ISO 8601 week directives are not interchangeable -with the year and week number directives above. Calling :meth:`~.datetime.strptime` with -incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. - The full set of format codes supported varies across platforms, because Python calls the platform C library's :c:func:`strftime` function, and platform variations are common. To see the full set of format codes supported on your @@ -2629,18 +2715,61 @@ differences between platforms in handling of unsupported format specifiers. ``%G``, ``%u`` and ``%V`` were added. .. versionadded:: 3.12 - ``%:z`` was added. + ``%:z`` was added for :meth:`~.datetime.strftime`. -Technical Detail +.. versionadded:: 3.15 + ``%D``, ``%F``, ``%n``, ``%t``, and ``%:z`` were added for + :meth:`~.datetime.strptime`. + + +Technical detail ^^^^^^^^^^^^^^^^ Broadly speaking, ``d.strftime(fmt)`` acts like the :mod:`time` module's ``time.strftime(fmt, d.timetuple())`` although not all objects support a :meth:`~date.timetuple` method. -For the :meth:`.datetime.strptime` class method, the default value is -``1900-01-01T00:00:00.000``: any components not specified in the format string -will be pulled from the default value. [#]_ +For the :meth:`.datetime.strptime` and :meth:`.date.strptime` class methods, +the default value is ``1900-01-01T00:00:00.000``: any components not specified +in the format string will be pulled from the default value. + +.. note:: + Format strings without separators can be ambiguous for parsing. For + example, with ``%Y%m%d``, the string ``2026111`` may be parsed either as + ``2026-11-01`` or as ``2026-01-11``. + Use separators to ensure the input is parsed as intended. + +.. note:: + When used to parse partial dates lacking a year, :meth:`.datetime.strptime` + and :meth:`.date.strptime` will raise when encountering February 29 because + the default year of 1900 is *not* a leap year. Always add a default leap + year to partial date strings before parsing. + +.. testsetup:: + + # doctest seems to turn the warning into an error which makes it + # show up and require matching and prevents the actual interesting + # exception from being raised. + # Manually apply the catch_warnings context manager + import warnings + catch_warnings = warnings.catch_warnings() + catch_warnings.__enter__() + warnings.simplefilter("ignore") + +.. testcleanup:: + + catch_warnings.__exit__() + +.. doctest:: + + >>> import datetime as dt + >>> value = "2/29" + >>> dt.datetime.strptime(value, "%m/%d") + Traceback (most recent call last): + ... + ValueError: day 29 must be in range 1..28 for month 2 in year 1900 + >>> dt.datetime.strptime(f"1904 {value}", "%Y %m/%d") + datetime.datetime(1904, 2, 29, 0, 0) Using ``datetime.strptime(date_string, format)`` is equivalent to:: @@ -2666,6 +2795,9 @@ an empty string instead. Notes: +(0) + This format code is currently unsupported by :meth:`~.datetime.strptime`. + (1) Because the format depends on the current locale, care should be taken when making assumptions about the output value. Field orderings will vary (for @@ -2724,12 +2856,18 @@ Notes: When the ``%z`` directive is provided to the :meth:`~.datetime.strptime` method, the UTC offsets can have a colon as a separator between hours, minutes and seconds. - For example, ``'+01:00:00'`` will be parsed as an offset of one hour. - In addition, providing ``'Z'`` is identical to ``'+00:00'``. + For example, both ``'+010000'`` and ``'+01:00:00'`` will be parsed as an offset + of one hour. In addition, providing ``'Z'`` is identical to ``'+00:00'``. ``%:z`` - Behaves exactly as ``%z``, but has a colon separator added between - hours, minutes and seconds. + When used with :meth:`~.datetime.strftime`, behaves exactly as ``%z``, + except that a colon separator is added between hours, minutes and seconds. + + When used with :meth:`~.datetime.strptime`, the UTC offset is *required* + to have a colon as a separator between hours, minutes and seconds. + For example, ``'+01:00:00'`` (but *not* ``'+010000'``) will be parsed as + an offset of one hour. In addition, providing ``'Z'`` is identical to + ``'+00:00'``. ``%Z`` In :meth:`~.datetime.strftime`, ``%Z`` is replaced by an empty string if @@ -2771,23 +2909,24 @@ Notes: include a year in the format. If the value you need to parse lacks a year, append an explicit dummy leap year. Otherwise your code will raise an exception when it encounters leap day because the default year used by the - parser is not a leap year. Users run into this bug every four years... + parser (1900) is not a leap year. Users run into that bug every leap year. .. doctest:: >>> month_day = "02/29" - >>> datetime.strptime(f"{month_day};1984", "%m/%d;%Y") # No leap year bug. + >>> dt.datetime.strptime(f"{month_day};1984", "%m/%d;%Y") # No leap year bug. datetime.datetime(1984, 2, 29, 0, 0) - .. deprecated-removed:: 3.13 3.15 + .. versionchanged:: 3.15 + Using ``%d`` without a year now raises :exc:`ValueError`. + + .. deprecated-removed:: 3.15 3.17 :meth:`~.datetime.strptime` calls using a format string containing - a day of month without a year now emit a - :exc:`DeprecationWarning`. In 3.15 or later we may change this into - an error or change the default year to a leap year. See :gh:`70647`. + ``%e`` without a year now emit a :exc:`DeprecationWarning`. .. rubric:: Footnotes -.. [#] If, that is, we ignore the effects of Relativity +.. [#] If, that is, we ignore the effects of relativity. .. [#] This matches the definition of the "proleptic Gregorian" calendar in Dershowitz and Reingold's book *Calendrical Calculations*, @@ -2798,5 +2937,3 @@ Notes: .. [#] See R. H. van Gent's `guide to the mathematics of the ISO 8601 calendar `_ for a good explanation. - -.. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since 1900 is not a leap year. diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 140ca5c1e34..646981e8692 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -8,7 +8,7 @@ -------------- -:mod:`dbm` is a generic interface to variants of the DBM database: +:mod:`!dbm` is a generic interface to variants of the DBM database: * :mod:`dbm.sqlite3` * :mod:`dbm.gnu` @@ -107,7 +107,7 @@ will automatically close them when done. .. versionchanged:: 3.2 :meth:`!get` and :meth:`!setdefault` methods are now available for all - :mod:`dbm` backends. + :mod:`!dbm` backends. .. versionchanged:: 3.4 Added native support for the context management protocol to the objects @@ -118,7 +118,7 @@ will automatically close them when done. instead of :exc:`KeyError`. .. versionchanged:: 3.13 - :meth:`!clear` methods are now available for all :mod:`dbm` backends. + :meth:`!clear` methods are now available for all :mod:`!dbm` backends. The following example records some hostnames and a corresponding title, and @@ -157,11 +157,10 @@ then prints out the contents of the database:: The individual submodules are described in the following sections. -:mod:`dbm.sqlite3` --- SQLite backend for dbm ---------------------------------------------- +:mod:`!dbm.sqlite3` --- SQLite backend for dbm +---------------------------------------------- .. module:: dbm.sqlite3 - :platform: All :synopsis: SQLite backend for dbm .. versionadded:: 3.13 @@ -171,8 +170,8 @@ The individual submodules are described in the following sections. -------------- This module uses the standard library :mod:`sqlite3` module to provide an -SQLite backend for the :mod:`dbm` module. -The files created by :mod:`dbm.sqlite3` can thus be opened by :mod:`sqlite3`, +SQLite backend for the :mod:`!dbm` module. +The files created by :mod:`!dbm.sqlite3` can thus be opened by :mod:`sqlite3`, or any other SQLite browser, including the SQLite CLI. .. include:: ../includes/wasm-notavail.rst @@ -215,36 +214,37 @@ or any other SQLite browser, including the SQLite CLI. .. note:: While reorganizing, as much as two times the size of the original database is required - in free disk space. However, be aware that this factor changes for each :mod:`dbm` submodule. + in free disk space. However, be aware that this factor changes for each :mod:`!dbm` submodule. - .. versionadded:: next + .. versionadded:: 3.15 -:mod:`dbm.gnu` --- GNU database manager ---------------------------------------- +:mod:`!dbm.gnu` --- GNU database manager +---------------------------------------- .. module:: dbm.gnu - :platform: Unix :synopsis: GNU database manager **Source code:** :source:`Lib/dbm/gnu.py` -------------- -The :mod:`dbm.gnu` module provides an interface to the :abbr:`GDBM (GNU dbm)` +The :mod:`!dbm.gnu` module provides an interface to the :abbr:`GDBM (GNU dbm)` library, similar to the :mod:`dbm.ndbm` module, but with additional functionality like crash tolerance. .. note:: - The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible + The file formats created by :mod:`!dbm.gnu` and :mod:`dbm.ndbm` are incompatible and can not be used interchangeably. .. include:: ../includes/wasm-mobile-notavail.rst +.. availability:: Unix. + .. exception:: error - Raised on :mod:`dbm.gnu`-specific errors, such as I/O errors. :exc:`KeyError` is + Raised on :mod:`!dbm.gnu`-specific errors, such as I/O errors. :exc:`KeyError` is raised for general mapping errors like specifying an incorrect key. @@ -275,9 +275,6 @@ functionality like crash tolerance. * ``'s'``: Synchronized mode. Changes to the database will be written immediately to the file. * ``'u'``: Do not lock database. - * ``'m'``: Do not use :manpage:`mmap(2)`. - This may harm performance, but improve crash tolerance. - .. versionadded:: next Not all flags are valid for all versions of GDBM. See the :data:`open_flags` member for a list of supported flag characters. @@ -338,7 +335,7 @@ functionality like crash tolerance. .. note:: While reorganizing, as much as one time the size of the original database is required - in free disk space. However, be aware that this factor changes for each :mod:`dbm` submodule. + in free disk space. However, be aware that this factor changes for each :mod:`!dbm` submodule. .. method:: gdbm.sync() @@ -346,25 +343,24 @@ functionality like crash tolerance. unwritten data to be written to the disk. -:mod:`dbm.ndbm` --- New Database Manager ----------------------------------------- +:mod:`!dbm.ndbm` --- New Database Manager +----------------------------------------- .. module:: dbm.ndbm - :platform: Unix :synopsis: The New Database Manager **Source code:** :source:`Lib/dbm/ndbm.py` -------------- -The :mod:`dbm.ndbm` module provides an interface to the +The :mod:`!dbm.ndbm` module provides an interface to the :abbr:`NDBM (New Database Manager)` library. This module can be used with the "classic" NDBM interface or the :abbr:`GDBM (GNU dbm)` compatibility interface. .. note:: - The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible + The file formats created by :mod:`dbm.gnu` and :mod:`!dbm.ndbm` are incompatible and can not be used interchangeably. .. warning:: @@ -376,9 +372,11 @@ This module can be used with the "classic" NDBM interface or the .. include:: ../includes/wasm-mobile-notavail.rst +.. availability:: Unix. + .. exception:: error - Raised on :mod:`dbm.ndbm`-specific errors, such as I/O errors. :exc:`KeyError` is raised + Raised on :mod:`!dbm.ndbm`-specific errors, such as I/O errors. :exc:`KeyError` is raised for general mapping errors like specifying an incorrect key. @@ -428,8 +426,8 @@ This module can be used with the "classic" NDBM interface or the Close the NDBM database. -:mod:`dbm.dumb` --- Portable DBM implementation ------------------------------------------------ +:mod:`!dbm.dumb` --- Portable DBM implementation +------------------------------------------------ .. module:: dbm.dumb :synopsis: Portable implementation of the simple DBM interface. @@ -440,23 +438,23 @@ This module can be used with the "classic" NDBM interface or the .. note:: - The :mod:`dbm.dumb` module is intended as a last resort fallback for the - :mod:`dbm` module when a more robust module is not available. The :mod:`dbm.dumb` + The :mod:`!dbm.dumb` module is intended as a last resort fallback for the + :mod:`!dbm` module when a more robust module is not available. The :mod:`!dbm.dumb` module is not written for speed and is not nearly as heavily used as the other database modules. -------------- -The :mod:`dbm.dumb` module provides a persistent :class:`dict`-like +The :mod:`!dbm.dumb` module provides a persistent :class:`dict`-like interface which is written entirely in Python. -Unlike other :mod:`dbm` backends, such as :mod:`dbm.gnu`, no +Unlike other :mod:`!dbm` backends, such as :mod:`dbm.gnu`, no external library is required. The :mod:`!dbm.dumb` module defines the following: .. exception:: error - Raised on :mod:`dbm.dumb`-specific errors, such as I/O errors. :exc:`KeyError` is + Raised on :mod:`!dbm.dumb`-specific errors, such as I/O errors. :exc:`KeyError` is raised for general mapping errors like specifying an incorrect key. @@ -487,7 +485,7 @@ The :mod:`!dbm.dumb` module defines the following: Python's AST compiler. .. warning:: - :mod:`dbm.dumb` does not support concurrent read/write access. (Multiple + :mod:`!dbm.dumb` does not support concurrent read/write access. (Multiple simultaneous read accesses are safe.) When a program has the database open for writing, no other program should have it open for reading or writing. @@ -520,9 +518,9 @@ The :mod:`!dbm.dumb` module defines the following: .. note:: While reorganizing, no additional free disk space is required. However, be aware - that this factor changes for each :mod:`dbm` submodule. + that this factor changes for each :mod:`!dbm` submodule. - .. versionadded:: next + .. versionadded:: 3.15 .. method:: dumbdbm.sync() diff --git a/Doc/library/debug.rst b/Doc/library/debug.rst index 60223657a44..f87c2481fb8 100644 --- a/Doc/library/debug.rst +++ b/Doc/library/debug.rst @@ -1,5 +1,5 @@ *********************** -Debugging and Profiling +Debugging and profiling *********************** These libraries help you with Python development: the debugger enables you to @@ -15,7 +15,8 @@ intrusive debugging or patching. bdb.rst faulthandler.rst pdb.rst - profile.rst + profiling.rst + pstats.rst timeit.rst trace.rst tracemalloc.rst diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 0b99a832405..2af5dfce961 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -4,14 +4,6 @@ .. module:: decimal :synopsis: Implementation of the General Decimal Arithmetic Specification. -.. moduleauthor:: Eric Price -.. moduleauthor:: Facundo Batista -.. moduleauthor:: Raymond Hettinger -.. moduleauthor:: Aahz -.. moduleauthor:: Tim Peters -.. moduleauthor:: Stefan Krah -.. sectionauthor:: Raymond D. Hettinger - **Source code:** :source:`Lib/decimal.py` .. import modules for testing inline doctests with the Sphinx doctest builder @@ -30,14 +22,16 @@ -------------- -The :mod:`decimal` module provides support for fast correctly rounded +The :mod:`!decimal` module provides support for fast correctly rounded decimal floating-point arithmetic. It offers several advantages over the :class:`float` datatype: -* Decimal "is based on a floating-point model which was designed with people - in mind, and necessarily has a paramount guiding principle -- computers must - provide an arithmetic that works in the same way as the arithmetic that - people learn at school." -- excerpt from the decimal arithmetic specification. +* Decimal "is based on a `floating-point model + `__ which was designed + with people in mind, and necessarily has a paramount guiding principle -- + computers must provide an arithmetic that works in the same way as the + arithmetic that people learn at school." -- excerpt from the decimal + arithmetic specification. * Decimal numbers can be represented exactly. In contrast, numbers like ``1.1`` and ``2.2`` do not have exact representations in binary @@ -238,6 +232,26 @@ floating-point flying circus: >>> c % a Decimal('0.77') +Decimals can be formatted (with :func:`format` built-in or :ref:`f-strings`) in +fixed-point or scientific notation, using the same formatting syntax (see +:ref:`formatspec`) as builtin :class:`float` type: + +.. doctest:: + + >>> format(Decimal('2.675'), "f") + '2.675' + >>> format(Decimal('2.675'), ".2f") + '2.68' + >>> f"{Decimal('2.675'):.2f}" + '2.68' + >>> format(Decimal('2.675'), ".2e") + '2.68e+0' + >>> with localcontext() as ctx: + ... ctx.rounding = ROUND_DOWN + ... print(format(Decimal('2.675'), ".2f")) + ... + 2.67 + And some mathematical functions are also available to Decimal: >>> getcontext().prec = 28 @@ -264,10 +278,10 @@ allows the settings to be changed. This approach meets the needs of most applications. For more advanced work, it may be useful to create alternate contexts using the -Context() constructor. To make an alternate active, use the :func:`setcontext` +:meth:`Context` constructor. To make an alternate active, use the :func:`setcontext` function. -In accordance with the standard, the :mod:`decimal` module provides two ready to +In accordance with the standard, the :mod:`!decimal` module provides two ready to use standard contexts, :const:`BasicContext` and :const:`ExtendedContext`. The former is especially useful for debugging because many of the traps are enabled: @@ -1569,7 +1583,16 @@ In addition to the three supplied contexts, new contexts can be created with the Constants --------- -The constants in this section are only relevant for the C module. They +.. data:: SPEC_VERSION + + The highest version of the General Decimal Arithmetic + Specification that this implementation complies with. + See https://speleotrove.com/decimal/decarith.html for the specification. + + .. versionadded:: 3.15 + + +The following constants are only relevant for the C module. They are also included in the pure Python version for compatibility. +---------------------------------+---------------------+-------------------------------+ @@ -1816,7 +1839,7 @@ properties of addition: >>> u * (v+w) Decimal('0.0060000') -The :mod:`decimal` module makes it possible to restore the identities by +The :mod:`!decimal` module makes it possible to restore the identities by expanding the precision sufficiently to avoid loss of significance: .. doctest:: newcontext @@ -1838,7 +1861,7 @@ expanding the precision sufficiently to avoid loss of significance: Special values ^^^^^^^^^^^^^^ -The number system for the :mod:`decimal` module provides special values +The number system for the :mod:`!decimal` module provides special values including ``NaN``, ``sNaN``, ``-Infinity``, ``Infinity``, and two zeros, ``+0`` and ``-0``. @@ -2100,20 +2123,20 @@ to work with the :class:`Decimal` class:: Decimal FAQ ----------- -Q. It is cumbersome to type ``decimal.Decimal('1234.5')``. Is there a way to +Q: It is cumbersome to type ``decimal.Decimal('1234.5')``. Is there a way to minimize typing when using the interactive interpreter? -A. Some users abbreviate the constructor to just a single letter: +A: Some users abbreviate the constructor to just a single letter: >>> D = decimal.Decimal >>> D('1.23') + D('3.45') Decimal('4.68') -Q. In a fixed-point application with two decimal places, some inputs have many +Q: In a fixed-point application with two decimal places, some inputs have many places and need to be rounded. Others are not supposed to have excess digits and need to be validated. What methods should be used? -A. The :meth:`~Decimal.quantize` method rounds to a fixed number of decimal places. If +A: The :meth:`~Decimal.quantize` method rounds to a fixed number of decimal places. If the :const:`Inexact` trap is set, it is also useful for validation: >>> TWOPLACES = Decimal(10) ** -2 # same as Decimal('0.01') @@ -2131,10 +2154,10 @@ the :const:`Inexact` trap is set, it is also useful for validation: ... Inexact: None -Q. Once I have valid two place inputs, how do I maintain that invariant +Q: Once I have valid two place inputs, how do I maintain that invariant throughout an application? -A. Some operations like addition, subtraction, and multiplication by an integer +A: Some operations like addition, subtraction, and multiplication by an integer will automatically preserve fixed point. Others operations, like division and non-integer multiplication, will change the number of decimal places and need to be followed-up with a :meth:`~Decimal.quantize` step: @@ -2166,21 +2189,21 @@ to handle the :meth:`~Decimal.quantize` step: >>> div(b, a) Decimal('0.03') -Q. There are many ways to express the same value. The numbers ``200``, +Q: There are many ways to express the same value. The numbers ``200``, ``200.000``, ``2E2``, and ``.02E+4`` all have the same value at various precisions. Is there a way to transform them to a single recognizable canonical value? -A. The :meth:`~Decimal.normalize` method maps all equivalent values to a single +A: The :meth:`~Decimal.normalize` method maps all equivalent values to a single representative: >>> values = map(Decimal, '200 200.000 2E2 .02E+4'.split()) >>> [v.normalize() for v in values] [Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2')] -Q. When does rounding occur in a computation? +Q: When does rounding occur in a computation? -A. It occurs *after* the computation. The philosophy of the decimal +A: It occurs *after* the computation. The philosophy of the decimal specification is that numbers are considered exact and are created independent of the current context. They can even have greater precision than current context. Computations process with those @@ -2198,10 +2221,10 @@ applied to the *result* of the computation:: >>> pi + 0 - Decimal('0.00005'). # Intermediate values are rounded Decimal('3.1416') -Q. Some decimal values always print with exponential notation. Is there a way +Q: Some decimal values always print with exponential notation. Is there a way to get a non-exponential representation? -A. For some values, exponential notation is the only way to express the number +A: For some values, exponential notation is the only way to express the number of significant places in the coefficient. For example, expressing ``5.0E+3`` as ``5000`` keeps the value constant but cannot show the original's two-place significance. @@ -2216,9 +2239,9 @@ value unchanged: >>> remove_exponent(Decimal('5E+3')) Decimal('5000') -Q. Is there a way to convert a regular float to a :class:`Decimal`? +Q: Is there a way to convert a regular float to a :class:`Decimal`? -A. Yes, any binary floating-point number can be exactly expressed as a +A: Yes, any binary floating-point number can be exactly expressed as a Decimal though an exact conversion may take more precision than intuition would suggest: @@ -2227,19 +2250,19 @@ suggest: >>> Decimal(math.pi) Decimal('3.141592653589793115997963468544185161590576171875') -Q. Within a complex calculation, how can I make sure that I haven't gotten a +Q: Within a complex calculation, how can I make sure that I haven't gotten a spurious result because of insufficient precision or rounding anomalies. -A. The decimal module makes it easy to test results. A best practice is to +A: The decimal module makes it easy to test results. A best practice is to re-run calculations using greater precision and with various rounding modes. Widely differing results indicate insufficient precision, rounding mode issues, ill-conditioned inputs, or a numerically unstable algorithm. -Q. I noticed that context precision is applied to the results of operations but +Q: I noticed that context precision is applied to the results of operations but not to the inputs. Is there anything to watch out for when mixing values of different precisions? -A. Yes. The principle is that all values are considered to be exact and so is +A: Yes. The principle is that all values are considered to be exact and so is the arithmetic on those values. Only the results are rounded. The advantage for inputs is that "what you type is what you get". A disadvantage is that the results can look odd if you forget that the inputs haven't been rounded: @@ -2267,9 +2290,9 @@ Alternatively, inputs can be rounded upon creation using the >>> Context(prec=5, rounding=ROUND_DOWN).create_decimal('1.2345678') Decimal('1.2345') -Q. Is the CPython implementation fast for large numbers? +Q: Is the CPython implementation fast for large numbers? -A. Yes. In the CPython and PyPy3 implementations, the C/CFFI versions of +A: Yes. In the CPython and PyPy3 implementations, the C/CFFI versions of the decimal module integrate the high speed `libmpdec `_ library for arbitrary precision correctly rounded decimal floating-point arithmetic [#]_. diff --git a/Doc/library/dialog.rst b/Doc/library/dialog.rst index e0693e8eb6e..5d522556235 100644 --- a/Doc/library/dialog.rst +++ b/Doc/library/dialog.rst @@ -1,18 +1,17 @@ Tkinter Dialogs =============== -:mod:`tkinter.simpledialog` --- Standard Tkinter input dialogs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:mod:`!tkinter.simpledialog` --- Standard Tkinter input dialogs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: tkinter.simpledialog - :platform: Tk :synopsis: Simple dialog windows **Source code:** :source:`Lib/tkinter/simpledialog.py` -------------- -The :mod:`tkinter.simpledialog` module contains convenience classes and +The :mod:`!tkinter.simpledialog` module contains convenience classes and functions for creating simple modal dialogs to get a value from the user. @@ -39,18 +38,17 @@ functions for creating simple modal dialogs to get a value from the user. -:mod:`tkinter.filedialog` --- File selection dialogs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:mod:`!tkinter.filedialog` --- File selection dialogs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: tkinter.filedialog - :platform: Tk :synopsis: Dialog classes for file selection **Source code:** :source:`Lib/tkinter/filedialog.py` -------------- -The :mod:`tkinter.filedialog` module provides classes and factory functions for +The :mod:`!tkinter.filedialog` module provides classes and factory functions for creating file/directory selection windows. Native Load/Save Dialogs @@ -204,18 +202,17 @@ These do not emulate the native look-and-feel of the platform. directory. Confirmation is required if an already existing file is selected. -:mod:`tkinter.commondialog` --- Dialog window templates -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:mod:`!tkinter.commondialog` --- Dialog window templates +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: tkinter.commondialog - :platform: Tk :synopsis: Tkinter base class for dialogs **Source code:** :source:`Lib/tkinter/commondialog.py` -------------- -The :mod:`tkinter.commondialog` module provides the :class:`Dialog` class that +The :mod:`!tkinter.commondialog` module provides the :class:`Dialog` class that is the base class for dialogs defined in other supporting modules. .. class:: Dialog(master=None, **options) diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index c55ecac3409..8b812c173b5 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -4,10 +4,6 @@ .. module:: difflib :synopsis: Helpers for computing differences between objects. -.. moduleauthor:: Tim Peters -.. sectionauthor:: Tim Peters -.. Markup by Fred L. Drake, Jr. - **Source code:** :source:`Lib/difflib.py` .. testsetup:: @@ -231,7 +227,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. *linejunk*: A function that accepts a single string argument, and returns true if the string is junk, or false if not. The default is ``None``. There is also a module-level function :func:`IS_LINE_JUNK`, which filters out lines - without visible characters, except for at most one pound character (``'#'``) + without visible characters, except for at most one hash character (``'#'``) -- however the underlying :class:`SequenceMatcher` class does a dynamic analysis of which lines are so frequent as to constitute noise, and this usually works better than using this function. @@ -323,7 +319,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. See :ref:`difflib-interface` for a more detailed example. - .. versionchanged:: next + .. versionchanged:: 3.15 Added the *color* parameter. @@ -366,7 +362,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. .. _sequence-matcher: -SequenceMatcher Objects +SequenceMatcher objects ----------------------- The :class:`SequenceMatcher` class has this constructor: @@ -594,7 +590,7 @@ are always at least as large as :meth:`~SequenceMatcher.ratio`: .. _sequencematcher-examples: -SequenceMatcher Examples +SequenceMatcher examples ------------------------ This example compares two strings, considering blanks to be "junk": @@ -645,7 +641,7 @@ If you want to know how to change the first sequence into the second, use .. _differ-objects: -Differ Objects +Differ objects -------------- Note that :class:`Differ`\ -generated deltas make no claim to be **minimal** @@ -694,7 +690,7 @@ The :class:`Differ` class has this constructor: .. _differ-examples: -Differ Example +Differ example -------------- This example compares two texts. First we set up the texts, sequences of diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 21bd1db5f33..3e7ae509fed 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -14,7 +14,7 @@ -------------- -The :mod:`dis` module supports the analysis of CPython :term:`bytecode` by +The :mod:`!dis` module supports the analysis of CPython :term:`bytecode` by disassembling it. The CPython bytecode which this module takes as an input is defined in the file :file:`Include/opcode.h` and used by the compiler and the interpreter. @@ -38,7 +38,7 @@ interpreter. Some instructions are accompanied by one or more inline cache entries, which take the form of :opcode:`CACHE` instructions. These instructions are hidden by default, but can be shown by passing ``show_caches=True`` to - any :mod:`dis` utility. Furthermore, the interpreter now adapts the + any :mod:`!dis` utility. Furthermore, the interpreter now adapts the bytecode to specialize it for different runtime conditions. The adaptive bytecode can be shown by passing ``adaptive=True``. @@ -87,7 +87,7 @@ the following command can be used to display the disassembly of Command-line interface ---------------------- -The :mod:`dis` module can be invoked as a script from the command line: +The :mod:`!dis` module can be invoked as a script from the command line: .. code-block:: sh @@ -223,7 +223,7 @@ Example: Analysis functions ------------------ -The :mod:`dis` module also defines the following analysis functions that convert +The :mod:`!dis` module also defines the following analysis functions that convert the input directly to the desired output. They can be useful if only a single operation is being performed, so the intermediate analysis object isn't useful: @@ -400,7 +400,7 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.10 The :pep:`626` :meth:`~codeobject.co_lines` method is used instead of the - :attr:`~codeobject.co_firstlineno` and :attr:`~codeobject.co_lnotab` + :attr:`~codeobject.co_firstlineno` and :attr:`!codeobject.co_lnotab` attributes of the :ref:`code object `. .. versionchanged:: 3.13 @@ -585,6 +585,22 @@ operations on it as if it was a Python list. The top of the stack corresponds to generate line tracing events. +.. opcode:: NOT_TAKEN + + Do nothing code. + Used by the interpreter to record :monitoring-event:`BRANCH_LEFT` + and :monitoring-event:`BRANCH_RIGHT` events for :mod:`sys.monitoring`. + + .. versionadded:: 3.14 + + +.. opcode:: POP_ITER + + Removes the iterator from the top of the stack. + + .. versionadded:: 3.14 + + .. opcode:: POP_TOP Removes the top-of-stack item:: @@ -752,7 +768,7 @@ not have to be) the original ``STACK[-2]``. end = STACK.pop() start = STACK.pop() container = STACK.pop() - values = STACK.pop() + value = STACK.pop() container[start:end] = value .. versionadded:: 3.12 @@ -1086,11 +1102,6 @@ iterations of the loop. Pushes ``co_consts[consti]`` onto the stack. -.. opcode:: LOAD_CONST_IMMORTAL (consti) - - Works as :opcode:`LOAD_CONST`, but is more efficient for immortal objects. - - .. opcode:: LOAD_SMALL_INT (i) Pushes the integer ``i`` onto the stack. @@ -1631,7 +1642,7 @@ iterations of the loop. Pushes a ``NULL`` to the stack. Used in the call sequence to match the ``NULL`` pushed by - :opcode:`LOAD_METHOD` for non-method calls. + :opcode:`LOAD_ATTR` for non-method calls. .. versionadded:: 3.11 @@ -1662,9 +1673,13 @@ iterations of the loop. * ``0x02`` a dictionary of keyword-only parameters' default values * ``0x04`` a tuple of strings containing parameters' annotations * ``0x08`` a tuple containing cells for free variables, making a closure + * ``0x10`` the :term:`annotate function` for the function object .. versionadded:: 3.13 + .. versionchanged:: 3.14 + Added ``0x10`` to indicate the annotate function for the function object. + .. opcode:: BUILD_SLICE (argc) @@ -1812,7 +1827,7 @@ iterations of the loop. ignore it. Before, only opcodes ``>= HAVE_ARGUMENT`` had an argument. .. versionchanged:: 3.12 - Pseudo instructions were added to the :mod:`dis` module, and for them + Pseudo instructions were added to the :mod:`!dis` module, and for them it is not true that comparison with ``HAVE_ARGUMENT`` indicates whether they use their arg. @@ -1952,14 +1967,15 @@ but are replaced by real opcodes or removed before bytecode is generated. Marks the end of the code block associated with the last ``SETUP_FINALLY``, ``SETUP_CLEANUP`` or ``SETUP_WITH``. + .. opcode:: JUMP -.. opcode:: JUMP_NO_INTERRUPT + JUMP_NO_INTERRUPT Undirected relative jump instructions which are replaced by their directed (forward/backward) counterparts by the assembler. .. opcode:: JUMP_IF_TRUE -.. opcode:: JUMP_IF_FALSE + JUMP_IF_FALSE Conditional jumps which do not impact the stack. Replaced by the sequence ``COPY 1``, ``TO_BOOL``, ``POP_JUMP_IF_TRUE/FALSE``. @@ -1975,12 +1991,6 @@ but are replaced by real opcodes or removed before bytecode is generated. This opcode is now a pseudo-instruction. -.. opcode:: LOAD_METHOD - - Optimized unbound method lookup. Emitted as a ``LOAD_ATTR`` opcode - with a flag set in the arg. - - .. _opcode_collections: Opcode collections diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 02b73ccd3f3..3298697af85 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -4,16 +4,11 @@ .. module:: doctest :synopsis: Test pieces of code within docstrings. -.. moduleauthor:: Tim Peters -.. sectionauthor:: Tim Peters -.. sectionauthor:: Moshe Zadka -.. sectionauthor:: Edward Loper - **Source code:** :source:`Lib/doctest.py` -------------- -The :mod:`doctest` module searches for pieces of text that look like interactive +The :mod:`!doctest` module searches for pieces of text that look like interactive Python sessions, and then executes those sessions to verify that they work exactly as shown. There are several common ways to use doctest: @@ -85,7 +80,7 @@ Here's a complete but small example module:: import doctest doctest.testmod() -If you run :file:`example.py` directly from the command line, :mod:`doctest` +If you run :file:`example.py` directly from the command line, :mod:`!doctest` works its magic: .. code-block:: shell-session @@ -94,7 +89,7 @@ works its magic: $ There's no output! That's normal, and it means all the examples worked. Pass -``-v`` to the script, and :mod:`doctest` prints a detailed log of what +``-v`` to the script, and :mod:`!doctest` prints a detailed log of what it's trying, and prints a summary at the end: .. code-block:: shell-session @@ -130,7 +125,7 @@ And so on, eventually ending with: Test passed. $ -That's all you need to know to start making productive use of :mod:`doctest`! +That's all you need to know to start making productive use of :mod:`!doctest`! Jump in. The following sections provide full details. Note that there are many examples of doctests in the standard Python test suite and libraries. Especially useful examples can be found in the standard test file @@ -252,7 +247,7 @@ For more information on :func:`testfile`, see section :ref:`doctest-basic-api`. Command-line Usage ------------------ -The :mod:`doctest` module can be invoked as a script from the command line: +The :mod:`!doctest` module can be invoked as a script from the command line: .. code-block:: bash @@ -450,7 +445,7 @@ The fine print: What's the Execution Context? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -By default, each time :mod:`doctest` finds a docstring to test, it uses a +By default, each time :mod:`!doctest` finds a docstring to test, it uses a *shallow copy* of :mod:`!M`'s globals, so that running tests doesn't change the module's real globals, and so that one test in :mod:`!M` can't leave behind crumbs that accidentally allow another test to work. This means examples can @@ -730,7 +725,7 @@ The second group of options controls how test failures are reported: There is also a way to register new option flag names, though this isn't -useful unless you intend to extend :mod:`doctest` internals via subclassing: +useful unless you intend to extend :mod:`!doctest` internals via subclassing: .. function:: register_optionflag(name) @@ -833,7 +828,7 @@ disabling an option via ``-`` in a directive can be useful. Warnings ^^^^^^^^ -:mod:`doctest` is serious about requiring exact matches in expected output. If +:mod:`!doctest` is serious about requiring exact matches in expected output. If even a single character doesn't match, the test fails. This will probably surprise you a few times, as you learn exactly what Python does and doesn't guarantee about output. For example, when printing a set, Python doesn't @@ -1035,7 +1030,7 @@ Unittest API ------------ As your collection of doctest'ed modules grows, you'll want a way to run all -their doctests systematically. :mod:`doctest` provides two functions that can +their doctests systematically. :mod:`!doctest` provides two functions that can be used to create :mod:`unittest` test suites from modules and text files containing doctests. To integrate with :mod:`unittest` test discovery, include a :ref:`load_tests ` function in your test module:: @@ -1123,7 +1118,7 @@ from text files and modules with doctests: The global ``__file__`` is added to the globals provided to doctests loaded from a text file using :func:`DocFileSuite`. - .. versionchanged:: next + .. versionchanged:: 3.15 Run each example as a :ref:`subtest `. @@ -1164,7 +1159,7 @@ from text files and modules with doctests: :func:`DocTestSuite` returns an empty :class:`unittest.TestSuite` if *module* contains no docstrings instead of raising :exc:`ValueError`. - .. versionchanged:: next + .. versionchanged:: 3.15 Run each example as a :ref:`subtest `. Under the covers, :func:`DocTestSuite` creates a :class:`unittest.TestSuite` out @@ -1179,7 +1174,7 @@ of :class:`!DocTestCase`. So both ways of creating a :class:`unittest.TestSuite` run instances of :class:`!DocTestCase`. This is important for a subtle reason: when you run -:mod:`doctest` functions yourself, you can control the :mod:`!doctest` options in +:mod:`!doctest` functions yourself, you can control the :mod:`!doctest` options in use directly, by passing option flags to :mod:`!doctest` functions. However, if you're writing a :mod:`unittest` framework, :mod:`!unittest` ultimately controls when and how tests get run. The framework author typically wants to control @@ -1187,13 +1182,13 @@ when and how tests get run. The framework author typically wants to control options), but there's no way to pass options through :mod:`!unittest` to :mod:`!doctest` test runners. -For this reason, :mod:`doctest` also supports a notion of :mod:`!doctest` +For this reason, :mod:`!doctest` also supports a notion of :mod:`!doctest` reporting flags specific to :mod:`unittest` support, via this function: .. function:: set_unittest_reportflags(flags) - Set the :mod:`doctest` reporting flags to use. + Set the :mod:`!doctest` reporting flags to use. Argument *flags* takes the :ref:`bitwise OR ` of option flags. See section :ref:`doctest-options`. Only "reporting flags" can be used. @@ -1564,7 +1559,7 @@ DocTestRunner objects containing *example*. *out* is the output function that was passed to :meth:`DocTestRunner.run`. - .. versionadded:: next + .. versionadded:: 3.15 .. method:: report_start(out, test, example) @@ -1923,7 +1918,7 @@ There are two exceptions that may be raised by :class:`DebugRunner` instances: Soapbox ------- -As mentioned in the introduction, :mod:`doctest` has grown to have three primary +As mentioned in the introduction, :mod:`!doctest` has grown to have three primary uses: #. Checking examples in docstrings. @@ -1941,7 +1936,7 @@ this that needs to be learned---it may not be natural at first. Examples should add genuine value to the documentation. A good example can often be worth many words. If done with care, the examples will be invaluable for your users, and will pay back the time it takes to collect them many times over as the years go -by and things change. I'm still amazed at how often one of my :mod:`doctest` +by and things change. I'm still amazed at how often one of my :mod:`!doctest` examples stops working after a "harmless" change. Doctest also makes an excellent tool for regression testing, especially if you diff --git a/Doc/library/email.charset.rst b/Doc/library/email.charset.rst index 6875af2be49..76a57031862 100644 --- a/Doc/library/email.charset.rst +++ b/Doc/library/email.charset.rst @@ -19,7 +19,7 @@ registry and several convenience methods for manipulating this registry. Instances of :class:`Charset` are used in several other modules within the :mod:`email` package. -Import this class from the :mod:`email.charset` module. +Import this class from the :mod:`!email.charset` module. .. class:: Charset(input_charset=DEFAULT_CHARSET) @@ -164,7 +164,7 @@ Import this class from the :mod:`email.charset` module. This method allows you to compare two :class:`Charset` instances for inequality. -The :mod:`email.charset` module also provides the following functions for adding +The :mod:`!email.charset` module also provides the following functions for adding new entries to the global character set, alias, and codec registries: diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst index b33fe82a6e4..04a41667f7d 100644 --- a/Doc/library/email.contentmanager.rst +++ b/Doc/library/email.contentmanager.rst @@ -4,9 +4,6 @@ .. module:: email.contentmanager :synopsis: Storing and Retrieving Content from MIME Parts -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray - **Source code:** :source:`Lib/email/contentmanager.py` ------------ diff --git a/Doc/library/email.encoders.rst b/Doc/library/email.encoders.rst index 9c8c8c9234e..1a9a1cad3a6 100644 --- a/Doc/library/email.encoders.rst +++ b/Doc/library/email.encoders.rst @@ -25,7 +25,7 @@ is especially true for :mimetype:`image/\*` and :mimetype:`text/\*` type message containing binary data. The :mod:`email` package provides some convenient encoders in its -:mod:`~email.encoders` module. These encoders are actually used by the +:mod:`!encoders` module. These encoders are actually used by the :class:`~email.mime.audio.MIMEAudio` and :class:`~email.mime.image.MIMEImage` class constructors to provide default encodings. All encoder functions take exactly one argument, the message object to encode. They usually extract the diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst index 689e7397cbc..2f7c9140cfc 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -8,7 +8,7 @@ -------------- -The following exception classes are defined in the :mod:`email.errors` module: +The following exception classes are defined in the :mod:`!email.errors` module: .. exception:: MessageError() diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index a3132d02687..6f4f813a0f8 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -232,7 +232,7 @@ a formatted string representation of a message object. For more detail, see :mod:`email.message`. -The :mod:`email.generator` module also provides a derived class, +The :mod:`!email.generator` module also provides a derived class, :class:`DecodedGenerator`, which is like the :class:`Generator` base class, except that non-\ :mimetype:`text` parts are not serialized, but are instead represented in the output stream by a string derived from a template filled diff --git a/Doc/library/email.header.rst b/Doc/library/email.header.rst index f49885b8785..e7e21d036e0 100644 --- a/Doc/library/email.header.rst +++ b/Doc/library/email.header.rst @@ -28,13 +28,13 @@ transferred using only 7-bit ASCII characters, so a slew of RFCs have been written describing how to encode email containing non-ASCII characters into :rfc:`2822`\ -compliant format. These RFCs include :rfc:`2045`, :rfc:`2046`, :rfc:`2047`, and :rfc:`2231`. The :mod:`email` package supports these standards -in its :mod:`email.header` and :mod:`email.charset` modules. +in its :mod:`!email.header` and :mod:`email.charset` modules. If you want to include non-ASCII characters in your email headers, say in the :mailheader:`Subject` or :mailheader:`To` fields, you should use the :class:`Header` class and assign the field in the :class:`~email.message.Message` object to an instance of :class:`Header` instead of using a string for the header -value. Import the :class:`Header` class from the :mod:`email.header` module. +value. Import the :class:`Header` class from the :mod:`!email.header` module. For example:: >>> from email.message import Message @@ -170,7 +170,7 @@ Here is the :class:`Header` class description: This method allows you to compare two :class:`Header` instances for inequality. -The :mod:`email.header` module also provides the following convenient functions. +The :mod:`!email.header` module also provides the following convenient functions. .. function:: decode_header(header) diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index 7f8044932fa..c6924a0ac29 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -4,9 +4,6 @@ .. module:: email.headerregistry :synopsis: Automatic Parsing of headers based on the field name -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray - **Source code:** :source:`Lib/email/headerregistry.py` -------------- @@ -269,6 +266,10 @@ variant, :attr:`~.BaseHeader.max_count` is set to 1. A dictionary mapping parameter names to parameter values. + .. versionchanged:: 3.15 + It is now a :class:`frozendict` instead of a + :class:`types.MappingProxyType`. + .. class:: ContentTypeHeader @@ -294,7 +295,7 @@ variant, :attr:`~.BaseHeader.max_count` is set to 1. ``inline`` and ``attachment`` are the only valid values in common use. -.. class:: ContentTransferEncoding +.. class:: ContentTransferEncodingHeader Handles the :mailheader:`Content-Transfer-Encoding` header. diff --git a/Doc/library/email.iterators.rst b/Doc/library/email.iterators.rst index 090981d84b4..ed300cdb30f 100644 --- a/Doc/library/email.iterators.rst +++ b/Doc/library/email.iterators.rst @@ -10,7 +10,7 @@ Iterating over a message object tree is fairly easy with the :meth:`Message.walk ` method. The -:mod:`email.iterators` module provides some useful higher level iterations over +:mod:`!email.iterators` module provides some useful higher level iterations over message object trees. diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index 71d6e321f38..b70df130e06 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -3,9 +3,6 @@ .. module:: email.message :synopsis: The base class representing email messages. -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray , - Barry A. Warsaw **Source code:** :source:`Lib/email/message.py` @@ -14,7 +11,7 @@ .. versionadded:: 3.6 [1]_ The central class in the :mod:`email` package is the :class:`EmailMessage` -class, imported from the :mod:`email.message` module. It is the base class for +class, imported from the :mod:`!email.message` module. It is the base class for the :mod:`email` object model. :class:`EmailMessage` provides the core functionality for setting and querying header fields, for accessing message bodies, and for creating or modifying structured messages. @@ -57,7 +54,7 @@ message objects. :class:`~email.policy.default` policy, which follows the rules of the email RFCs except for line endings (instead of the RFC mandated ``\r\n``, it uses the Python standard ``\n`` line endings). For more information see the - :mod:`~email.policy` documentation. + :mod:`~email.policy` documentation. [2]_ .. method:: as_string(unixfrom=False, maxheaderlen=None, policy=None) @@ -749,3 +746,9 @@ message objects. .. [1] Originally added in 3.4 as a :term:`provisional module `. Docs for legacy message class moved to :ref:`compat32_message`. + +.. [2] The :class:`EmailMessage` class requires a policy that provides a + ``content_manager`` attribute for content management methods like + ``set_content()`` and ``get_content()`` to work. The legacy + :const:`~email.policy.compat32` policy does not support these methods + and should not be used with :class:`EmailMessage`. diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index 6a70714dc3e..6a67bf7c8e5 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -125,10 +125,10 @@ Here is the API for the :class:`BytesFeedParser`: Parser API ^^^^^^^^^^ -The :class:`BytesParser` class, imported from the :mod:`email.parser` module, +The :class:`BytesParser` class, imported from the :mod:`!email.parser` module, provides an API that can be used to parse a message when the complete contents of the message are available in a :term:`bytes-like object` or file. The -:mod:`email.parser` module also provides :class:`Parser` for parsing strings, +:mod:`!email.parser` module also provides :class:`Parser` for parsing strings, and header-only parsers, :class:`BytesHeaderParser` and :class:`HeaderParser`, which can be used if you're only interested in the headers of the message. :class:`BytesHeaderParser` and :class:`HeaderParser` @@ -155,7 +155,7 @@ message body, instead setting the payload to the raw body. Read all the data from the binary file-like object *fp*, parse the resulting bytes, and return the message object. *fp* must support - both the :meth:`~io.IOBase.readline` and the :meth:`~io.IOBase.read` + both the :meth:`~io.IOBase.readline` and the :meth:`~io.BufferedIOBase.read` methods. The bytes contained in *fp* must be formatted as a block of :rfc:`5322` diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index 6b997ee784f..816d02d86f4 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -4,9 +4,6 @@ .. module:: email.policy :synopsis: Controlling the parsing and generating of messages -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray - .. versionadded:: 3.3 **Source code:** :source:`Lib/email/policy.py` @@ -406,11 +403,26 @@ added matters. To illustrate:: .. attribute:: utf8 If ``False``, follow :rfc:`5322`, supporting non-ASCII characters in - headers by encoding them as "encoded words". If ``True``, follow - :rfc:`6532` and use ``utf-8`` encoding for headers. Messages + headers by encoding them as :rfc:`2047` "encoded words". If ``True``, + follow :rfc:`6532` and use ``utf-8`` encoding for headers. Messages formatted in this way may be passed to SMTP servers that support the ``SMTPUTF8`` extension (:rfc:`6531`). + When ``False``, the generator will raise + :exc:`~email.errors.HeaderWriteError` if any header includes non-ASCII + characters in a context where :rfc:`2047` does not permit encoded words. + This particularly applies to mailboxes ("addr-spec") with non-ASCII + characters, which can be created via + :class:`~email.headerregistry.Address`. To use a mailbox with a non-ASCII + domain name with ``utf8=False``, first encode the domain using the + third-party :pypi:`idna` or :pypi:`uts46` module or with + :mod:`encodings.idna`. It is not possible to use a non-ASCII username + ("local-part") in a mailbox when ``utf8=False``. + + .. versionchanged:: 3.15 + Can trigger the raising of :exc:`~email.errors.HeaderWriteError`. + (Earlier versions incorrectly applied :rfc:`2047` in certain contexts, + mostly notably in addr-specs.) .. attribute:: refold_source @@ -602,7 +614,7 @@ The header objects and their attributes are described in This concrete :class:`Policy` is the backward compatibility policy. It replicates the behavior of the email package in Python 3.2. The - :mod:`~email.policy` module also defines an instance of this class, + :mod:`!policy` module also defines an instance of this class, :const:`compat32`, that is used as the default policy. Thus the default behavior of the email package is to maintain compatibility with Python 3.2. @@ -662,6 +674,13 @@ The header objects and their attributes are described in An instance of :class:`Compat32`, providing backward compatibility with the behavior of the email package in Python 3.2. + .. note:: + + The :const:`compat32` policy should not be used as a policy for + :class:`~email.message.EmailMessage` objects, and should only be used + to serialize messages that were created using the :const:`compat32` + policy. + .. rubric:: Footnotes diff --git a/Doc/library/email.rst b/Doc/library/email.rst index 66c42e4a500..98b47ffd740 100644 --- a/Doc/library/email.rst +++ b/Doc/library/email.rst @@ -4,18 +4,15 @@ .. module:: email :synopsis: Package supporting the parsing, manipulating, and generating email messages. -.. moduleauthor:: Barry A. Warsaw , - R. David Murray -.. sectionauthor:: R. David Murray **Source code:** :source:`Lib/email/__init__.py` -------------- -The :mod:`email` package is a library for managing email messages. It is +The :mod:`!email` package is a library for managing email messages. It is specifically *not* designed to do any sending of email messages to SMTP (:rfc:`2821`), NNTP, or other servers; those are functions of modules such as -:mod:`smtplib`. The :mod:`email` package attempts to be as +:mod:`smtplib`. The :mod:`!email` package attempts to be as RFC-compliant as possible, supporting :rfc:`5322` and :rfc:`6532`, as well as such MIME-related RFCs as :rfc:`2045`, :rfc:`2046`, :rfc:`2047`, :rfc:`2183`, and :rfc:`2231`. @@ -68,7 +65,7 @@ high level structure in question, and not the details of how those structures are represented. Since MIME content types are used widely in modern internet software (not just email), this will be a familiar concept to many programmers. -The following sections describe the functionality of the :mod:`email` package. +The following sections describe the functionality of the :mod:`!email` package. We start with the :mod:`~email.message` object model, which is the primary interface an application will use, and follow that with the :mod:`~email.parser` and :mod:`~email.generator` components. Then we cover the @@ -102,7 +99,7 @@ compatibility reasons. :class:`~email.message.EmailMessage`/:class:`~email.policy.EmailPolicy` API. -Contents of the :mod:`email` package documentation: +Contents of the :mod:`!email` package documentation: .. toctree:: diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst index 611549604fd..e0d2c19a3b0 100644 --- a/Doc/library/email.utils.rst +++ b/Doc/library/email.utils.rst @@ -8,7 +8,7 @@ -------------- -There are a couple of useful utilities provided in the :mod:`email.utils` +There are a couple of useful utilities provided in the :mod:`!email.utils` module: .. function:: localtime(dt=None) diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index fa102c4a080..e0d77229b11 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`ensurepip` package provides support for bootstrapping the ``pip`` +The :mod:`!ensurepip` package provides support for bootstrapping the ``pip`` installer into an existing Python installation or virtual environment. This bootstrapping approach reflects the fact that ``pip`` is an independent project with its own release cycle, and the latest available stable version @@ -30,6 +30,8 @@ when creating a virtual environment) or after explicitly uninstalling needed to bootstrap ``pip`` are included as internal parts of the package. +.. include:: ../includes/optional-module.rst + .. seealso:: :ref:`installing-index` @@ -40,7 +42,9 @@ when creating a virtual environment) or after explicitly uninstalling .. include:: ../includes/wasm-mobile-notavail.rst -Command line interface +.. _ensurepip-cli: + +Command-line interface ---------------------- .. program:: ensurepip @@ -95,7 +99,7 @@ Providing both of the script selection options will trigger an exception. Module API ---------- -:mod:`ensurepip` exposes two functions for programmatic use: +:mod:`!ensurepip` exposes two functions for programmatic use: .. function:: version() diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 8875669ffcc..be7f59b0fce 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -4,11 +4,6 @@ .. module:: enum :synopsis: Implementation of an enumeration class. -.. moduleauthor:: Ethan Furman -.. sectionauthor:: Barry Warsaw -.. sectionauthor:: Eli Bendersky -.. sectionauthor:: Ethan Furman - .. versionadded:: 3.4 **Source code:** :source:`Lib/enum.py` @@ -61,7 +56,7 @@ are not normal Python classes. See --------------- -Module Contents +Module contents --------------- :class:`EnumType` @@ -153,6 +148,12 @@ Module Contents Return a list of all power-of-two integers contained in a flag. + :func:`enum.bin` + + Like built-in :func:`bin`, except negative values are represented in + two's complement, and the leading bit always indicates sign + (``0`` implies positive, ``1`` implies negative). + .. versionadded:: 3.6 ``Flag``, ``IntFlag``, ``auto`` .. versionadded:: 3.11 ``StrEnum``, ``EnumCheck``, ``ReprEnum``, ``FlagBoundary``, ``property``, ``member``, ``nonmember``, ``global_enum``, ``show_flag_values`` @@ -160,7 +161,7 @@ Module Contents --------------- -Data Types +Data types ---------- @@ -239,7 +240,7 @@ Data Types .. method:: EnumType.__len__(cls) - Returns the number of member in *cls*:: + Returns the number of members in *cls*:: >>> len(Color) 3 @@ -301,6 +302,28 @@ Data Types No longer used, kept for backward compatibility. (class attribute, removed during class creation). + The :attr:`~Enum._order_` attribute can be provided to help keep Python 2 / Python 3 code in sync. + It will be checked against the actual order of the enumeration and raise an error if the two do not match:: + + >>> class Color(Enum): + ... _order_ = 'RED GREEN BLUE' + ... RED = 1 + ... BLUE = 3 + ... GREEN = 2 + ... + Traceback (most recent call last): + ... + TypeError: member order does not match _order_: + ['RED', 'BLUE', 'GREEN'] + ['RED', 'GREEN', 'BLUE'] + + .. note:: + + In Python 2 code the :attr:`~Enum._order_` attribute is necessary as definition + order is lost before it can be recorded. + + .. versionadded:: 3.6 + .. attribute:: Enum._ignore_ ``_ignore_`` is only used during creation and is removed from the @@ -310,12 +333,15 @@ Data Types names will also be removed from the completed enumeration. See :ref:`TimePeriod ` for an example. + .. versionadded:: 3.7 + .. method:: Enum.__dir__(self) Returns ``['__class__', '__doc__', '__module__', 'name', 'value']`` and any public methods defined on *self.__class__*:: - >>> from datetime import date + >>> from enum import Enum + >>> import datetime as dt >>> class Weekday(Enum): ... MONDAY = 1 ... TUESDAY = 2 @@ -326,7 +352,7 @@ Data Types ... SUNDAY = 7 ... @classmethod ... def today(cls): - ... print('today is %s' % cls(date.today().isoweekday()).name) + ... print(f'today is {cls(dt.date.today().isoweekday()).name}') ... >>> dir(Weekday.SATURDAY) ['__class__', '__doc__', '__eq__', '__hash__', '__module__', 'name', 'today', 'value'] @@ -339,9 +365,18 @@ Data Types :last_values: A list of the previous values. A *staticmethod* that is used to determine the next value returned by - :class:`auto`:: + :class:`auto`. - >>> from enum import auto + .. note:: + For standard :class:`Enum` classes the next value chosen is the highest + value seen incremented by one. + + For :class:`Flag` classes the next value chosen will be the next highest + power-of-two. + + This method may be overridden, e.g.:: + + >>> from enum import auto, Enum >>> class PowersOfThree(Enum): ... @staticmethod ... def _generate_next_value_(name, start, count, last_values): @@ -352,6 +387,10 @@ Data Types >>> PowersOfThree.SECOND.value 9 + .. versionadded:: 3.6 + .. versionchanged:: 3.13 + Prior versions would use the last seen value instead of the highest value. + .. method:: Enum.__init__(self, *args, **kwds) By default, does nothing. If multiple values are given in the member @@ -373,7 +412,7 @@ Data Types A *classmethod* for looking up values not found in *cls*. By default it does nothing, but can be overridden to implement custom search behavior:: - >>> from enum import StrEnum + >>> from enum import auto, StrEnum >>> class Build(StrEnum): ... DEBUG = auto() ... OPTIMIZED = auto() @@ -390,6 +429,8 @@ Data Types >>> Build('deBUG') + .. versionadded:: 3.6 + .. method:: Enum.__new__(cls, *args, **kwds) By default, doesn't exist. If specified, either in the enum class @@ -412,6 +453,7 @@ Data Types Returns the string used for *repr()* calls. By default, returns the *Enum* name, member name, and value, but can be overridden:: + >>> from enum import auto, Enum >>> class OtherStyle(Enum): ... ALTERNATE = auto() ... OTHER = auto() @@ -428,6 +470,7 @@ Data Types Returns the string used for *str()* calls. By default, returns the *Enum* name and member name, but can be overridden:: + >>> from enum import auto, Enum >>> class OtherStyle(Enum): ... ALTERNATE = auto() ... OTHER = auto() @@ -443,6 +486,7 @@ Data Types Returns the string used for *format()* and *f-string* calls. By default, returns :meth:`__str__` return value, but can be overridden:: + >>> from enum import auto, Enum >>> class OtherStyle(Enum): ... ALTERNATE = auto() ... OTHER = auto() @@ -458,7 +502,7 @@ Data Types Using :class:`auto` with :class:`Enum` results in integers of increasing value, starting with ``1``. - .. versionchanged:: 3.12 Added :ref:`enum-dataclass-support` + .. versionchanged:: 3.12 Added :ref:`enum-dataclass-support`. .. method:: Enum._add_alias_ @@ -480,7 +524,8 @@ Data Types >>> Color(42) - Raises a :exc:`ValueError` if the value is already linked with a different member. + | Raises a :exc:`ValueError` if the value is already linked with a different member. + | See :ref:`multi-value-enum` for an example. .. versionadded:: 3.13 @@ -879,6 +924,8 @@ Data Types --------------- +.. _enum-dunder-sunder: + Supported ``__dunder__`` names """""""""""""""""""""""""""""" @@ -886,7 +933,7 @@ Supported ``__dunder__`` names items. It is only available on the class. :meth:`~Enum.__new__`, if specified, must create and return the enum members; -it is also a very good idea to set the member's :attr:`!_value_` appropriately. +it is also a very good idea to set the member's :attr:`~Enum._value_` appropriately. Once all the members are created it is no longer used. @@ -902,17 +949,10 @@ Supported ``_sunder_`` names from the final class - :attr:`~Enum._order_` -- no longer used, kept for backward compatibility (class attribute, removed during class creation) + - :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for an enum member; may be overridden - .. note:: - - For standard :class:`Enum` classes the next value chosen is the highest - value seen incremented by one. - - For :class:`Flag` classes the next value chosen will be the next highest - power-of-two. - - :meth:`~Enum._add_alias_` -- adds a new name as an alias to an existing member. - :meth:`~Enum._add_value_alias_` -- adds a new value as an alias to an @@ -930,26 +970,27 @@ Supported ``_sunder_`` names --------------- -Utilities and Decorators +Utilities and decorators ------------------------ .. class:: auto *auto* can be used in place of a value. If used, the *Enum* machinery will call an :class:`Enum`'s :meth:`~Enum._generate_next_value_` to get an appropriate value. - For :class:`Enum` and :class:`IntEnum` that appropriate value will be the last value plus - one; for :class:`Flag` and :class:`IntFlag` it will be the first power-of-two greater - than the highest value; for :class:`StrEnum` it will be the lower-cased version of + For :class:`Enum` and :class:`IntEnum` that appropriate value will be the highest value seen + plus one; for :class:`Flag` and :class:`IntFlag` it will be the first power-of-two greater + than the highest value seen; for :class:`StrEnum` it will be the lower-cased version of the member's name. Care must be taken if mixing *auto()* with manually specified values. - *auto* instances are only resolved when at the top level of an assignment: + *auto* instances are only resolved when at the top level of an assignment, either by + itself or as part of a tuple: * ``FIRST = auto()`` will work (auto() is replaced with ``1``); * ``SECOND = auto(), -2`` will work (auto is replaced with ``2``, so ``2, -2`` is used to create the ``SECOND`` enum member; - * ``THREE = [auto(), -3]`` will *not* work (``, -3`` is used to - create the ``THREE`` enum member) + * ``THIRD = [auto(), -3]`` will *not* work (``[, -3]`` is used to + create the ``THIRD`` enum member) .. versionchanged:: 3.11.1 @@ -959,7 +1000,7 @@ Utilities and Decorators ``_generate_next_value_`` can be overridden to customize the values used by *auto*. - .. note:: in 3.13 the default ``_generate_next_value_`` will always return + .. note:: In version 3.13 the default ``_generate_next_value_`` will always return the highest member value incremented by 1, and will fail if any member is an incompatible type. @@ -969,7 +1010,7 @@ Utilities and Decorators enumerations. It allows member attributes to have the same names as members themselves. - .. note:: the *property* and the member must be defined in separate classes; + .. note:: The *property* and the member must be defined in separate classes; for example, the *value* and *name* attributes are defined in the *Enum* class, and *Enum* subclasses can define members with the names ``value`` and ``name``. @@ -1030,6 +1071,20 @@ Utilities and Decorators .. versionadded:: 3.11 +.. function:: bin(num, max_bits=None) + + Like built-in :func:`bin`, except negative values are represented in + two's complement, and the leading bit always indicates sign + (``0`` implies positive, ``1`` implies negative). + + >>> import enum + >>> enum.bin(10) + '0b0 1010' + >>> enum.bin(~10) # ~10 is -11 + '0b1 0101' + + .. versionadded:: 3.11 + --------------- Notes diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 89ebb69d931..7fc6055aa9a 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -221,7 +221,7 @@ The following exceptions are the exceptions that are usually raised. .. exception:: EOFError Raised when the :func:`input` function hits an end-of-file condition (EOF) - without reading any data. (Note: the :meth:`!io.IOBase.read` and + without reading any data. (Note: the :meth:`io.TextIOBase.read` and :meth:`io.IOBase.readline` methods return an empty string when they hit EOF.) @@ -266,6 +266,12 @@ The following exceptions are the exceptions that are usually raised. .. versionadded:: 3.6 +.. exception:: ImportCycleError + + A subclass of :exc:`ImportError` which is raised when a lazy import fails + because it (directly or indirectly) tries to import itself. + + .. versionadded:: 3.15 .. exception:: IndexError @@ -450,7 +456,7 @@ The following exceptions are the exceptions that are usually raised. :meth:`threading.Thread.join` can now raise this exception. - .. versionchanged:: next + .. versionchanged:: 3.15 This exception may be raised when acquiring :meth:`threading.Lock` or :meth:`threading.RLock`. @@ -742,8 +748,8 @@ depending on the system error code. .. attribute:: characters_written - An integer containing the number of characters written to the stream - before it blocked. This attribute is available when using the + An integer containing the number of **bytes** written to the stream + before it blocked. This attribute is available when using the buffered I/O classes from the :mod:`io` module. .. exception:: ChildProcessError @@ -978,6 +984,12 @@ their subgroups based on the types of the contained exceptions. raises a :exc:`TypeError` if any contained exception is not an :exc:`Exception` subclass. + .. impl-detail:: + + The ``excs`` parameter may be any sequence, but lists and tuples are + specifically processed more efficiently here. For optimal performance, + pass a tuple as ``excs``. + .. attribute:: message The ``msg`` argument to the constructor. This is a read-only attribute. diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index 677966a8b2e..529e97bae6d 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -31,7 +31,8 @@ tracebacks: * Each string is limited to 500 characters. * Only the filename, the function name and the line number are displayed. (no source code) -* It is limited to 100 frames and 100 threads. +* It is limited to 100 frames per thread, and 100 threads + (configurable via *max_threads*). * The order is reversed: the most recent call is shown first. By default, the Python traceback is written to :data:`sys.stderr`. To see @@ -55,16 +56,20 @@ at Python startup. Dumping the traceback --------------------- -.. function:: dump_traceback(file=sys.stderr, all_threads=True) +.. function:: dump_traceback(file=sys.stderr, all_threads=True, *, max_threads=100) Dump the tracebacks of all threads into *file*. If *all_threads* is - ``False``, dump only the current thread. + ``False``, dump only the current thread. *max_threads* caps the number + of threads dumped. .. seealso:: :func:`traceback.print_tb`, which can be used to print a traceback object. .. versionchanged:: 3.5 Added support for passing file descriptor to this function. + .. versionchanged:: next + Added the *max_threads* keyword argument. + Dumping the C stack ------------------- @@ -100,7 +105,7 @@ instead of the stack, even if the operating system supports dumping stacks. Fault handler state ------------------- -.. function:: enable(file=sys.stderr, all_threads=True, c_stack=True) +.. function:: enable(file=sys.stderr, all_threads=True, c_stack=True, *, max_threads=100) Enable the fault handler: install handlers for the :const:`~signal.SIGSEGV`, :const:`~signal.SIGFPE`, :const:`~signal.SIGABRT`, :const:`~signal.SIGBUS` @@ -116,6 +121,8 @@ Fault handler state traceback, unless the system does not support it. See :func:`dump_c_stack` for more information on compatibility. + *max_threads* caps the number of threads dumped when a fatal signal fires. + .. versionchanged:: 3.5 Added support for passing file descriptor to this function. @@ -133,6 +140,9 @@ Fault handler state .. versionchanged:: 3.14 The dump now displays the C stack trace if *c_stack* is true. + .. versionchanged:: next + Added the *max_threads* keyword argument. + .. function:: disable() Disable the fault handler: uninstall the signal handlers installed by @@ -146,7 +156,7 @@ Fault handler state Dumping the tracebacks after a timeout -------------------------------------- -.. function:: dump_traceback_later(timeout, repeat=False, file=sys.stderr, exit=False) +.. function:: dump_traceback_later(timeout, repeat=False, file=sys.stderr, exit=False, *, max_threads=100) Dump the tracebacks of all threads, after a timeout of *timeout* seconds, or every *timeout* seconds if *repeat* is ``True``. If *exit* is ``True``, call @@ -154,7 +164,7 @@ Dumping the tracebacks after a timeout :c:func:`!_exit` exits the process immediately, which means it doesn't do any cleanup like flushing file buffers.) If the function is called twice, the new call replaces previous parameters and resets the timeout. The timer has a - sub-second resolution. + sub-second resolution. *max_threads* caps the number of threads dumped. The *file* must be kept open until the traceback is dumped or :func:`cancel_dump_traceback_later` is called: see :ref:`issue with file @@ -168,6 +178,9 @@ Dumping the tracebacks after a timeout .. versionchanged:: 3.7 This function is now always available. + .. versionchanged:: next + Added the *max_threads* keyword argument. + .. function:: cancel_dump_traceback_later() Cancel the last call to :func:`dump_traceback_later`. @@ -176,11 +189,12 @@ Dumping the tracebacks after a timeout Dumping the traceback on a user signal -------------------------------------- -.. function:: register(signum, file=sys.stderr, all_threads=True, chain=False) +.. function:: register(signum, file=sys.stderr, all_threads=True, chain=False, *, max_threads=100) Register a user signal: install a handler for the *signum* signal to dump the traceback of all threads, or of the current thread if *all_threads* is ``False``, into *file*. Call the previous handler if chain is ``True``. + *max_threads* caps the number of threads dumped. The *file* must be kept open until the signal is unregistered by :func:`unregister`: see :ref:`issue with file descriptors `. @@ -190,6 +204,9 @@ Dumping the traceback on a user signal .. versionchanged:: 3.5 Added support for passing file descriptor to this function. + .. versionchanged:: next + Added the *max_threads* keyword argument. + .. function:: unregister(signum) Unregister a user signal: uninstall the handler of the *signum* signal diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index 5c078df44ff..c28e4d6c0cc 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -2,11 +2,8 @@ ========================================================== .. module:: fcntl - :platform: Unix :synopsis: The fcntl() and ioctl() system calls. -.. sectionauthor:: Jaap Vermeulen - .. index:: pair: UNIX; file control pair: UNIX; I/O control @@ -53,7 +50,7 @@ descriptor. the latter setting ``FD_CLOEXEC`` flag in addition. .. versionchanged:: 3.12 - On Linux >= 4.5, the :mod:`fcntl` module exposes the ``FICLONE`` and + On Linux >= 4.5, the :mod:`!fcntl` module exposes the ``FICLONE`` and ``FICLONERANGE`` constants, which allow to share some data of one file with another file by reflinking on some filesystems (e.g., btrfs, OCFS2, and XFS). This behavior is commonly referred to as "copy-on-write". @@ -91,7 +88,7 @@ The module defines the following functions: Perform the operation *cmd* on file descriptor *fd* (file objects providing a :meth:`~io.IOBase.fileno` method are accepted as well). The values used for *cmd* are operating system dependent, and are available as constants - in the :mod:`fcntl` module, using the same names as used in the relevant C + in the :mod:`!fcntl` module, using the same names as used in the relevant C header files. The argument *arg* can either be an integer value, a :term:`bytes-like object`, or a string. The type and size of *arg* must match the type and size of @@ -125,7 +122,7 @@ The module defines the following functions: Add support of arbitrary :term:`bytes-like objects `, not only :class:`bytes`. - .. versionchanged:: next + .. versionchanged:: 3.15 The size of bytes-like objects is no longer limited to 1024 bytes. @@ -187,7 +184,7 @@ The module defines the following functions: The GIL is always released during a system call. System calls failing with EINTR are automatically retried. - .. versionchanged:: next + .. versionchanged:: 3.15 The size of not mutated bytes-like objects is no longer limited to 1024 bytes. diff --git a/Doc/library/filecmp.rst b/Doc/library/filecmp.rst index abd1b8c826d..f8365b44c5a 100644 --- a/Doc/library/filecmp.rst +++ b/Doc/library/filecmp.rst @@ -4,17 +4,15 @@ .. module:: filecmp :synopsis: Compare files efficiently. -.. sectionauthor:: Moshe Zadka - **Source code:** :source:`Lib/filecmp.py` -------------- -The :mod:`filecmp` module defines functions to compare files and directories, +The :mod:`!filecmp` module defines functions to compare files and directories, with various optional time/correctness trade-offs. For comparing files, see also the :mod:`difflib` module. -The :mod:`filecmp` module defines the following functions: +The :mod:`!filecmp` module defines the following functions: .. function:: cmp(f1, f2, shallow=True) diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index 8f32b11e565..5be16797be9 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -4,9 +4,6 @@ .. module:: fileinput :synopsis: Loop over standard input or a list of files. -.. moduleauthor:: Guido van Rossum -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/fileinput.py` -------------- diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index ee654b7a83e..a213679e4e2 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -103,7 +103,8 @@ functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`, :func:`.filter .. function:: translate(pat) Return the shell-style pattern *pat* converted to a regular expression for - using with :func:`re.match`. The pattern is expected to be a :class:`str`. + using with :func:`re.prefixmatch`. The pattern is expected to be a + :class:`str`. Example: @@ -113,7 +114,7 @@ functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`, :func:`.filter >>> regex '(?s:.*\\.txt)\\z' >>> reobj = re.compile(regex) - >>> reobj.match('foobar.txt') + >>> reobj.prefixmatch('foobar.txt') diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index d6d1c7a461c..b02e7b5b641 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -4,14 +4,11 @@ .. module:: fractions :synopsis: Rational numbers. -.. moduleauthor:: Jeffrey Yasskin -.. sectionauthor:: Jeffrey Yasskin - **Source code:** :source:`Lib/fractions.py` -------------- -The :mod:`fractions` module provides support for rational number arithmetic. +The :mod:`!fractions` module provides support for rational number arithmetic. A Fraction instance can be constructed from a pair of rational numbers, from diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index bb153220672..e1baeff3f37 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -23,7 +23,7 @@ The default encoding is UTF-8, following :rfc:`2640`. .. include:: ../includes/wasm-notavail.rst -Here's a sample session using the :mod:`ftplib` module:: +Here's a sample session using the :mod:`!ftplib` module:: >>> from ftplib import FTP >>> ftp = FTP('ftp.us.debian.org') # connect to host, default port @@ -524,14 +524,9 @@ FTP_TLS objects :class:`!FTP_TLS` class inherits from :class:`FTP`, defining these additional methods and attributes: - .. attribute:: FTP_TLS.ssl_version - - The SSL version to use (defaults to :data:`ssl.PROTOCOL_SSLv23`). - .. method:: FTP_TLS.auth() - Set up a secure control connection by using TLS or SSL, depending on what - is specified in the :attr:`ssl_version` attribute. + Set up a secure control connection by using TLS. .. versionchanged:: 3.4 The method now supports hostname check with @@ -548,7 +543,7 @@ FTP_TLS objects .. method:: FTP_TLS.prot_p() - Set up secure data connection. + Set up secure data connection by using TLS. .. method:: FTP_TLS.prot_c() diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 61799e303a1..06fd5cdc7be 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -19,13 +19,13 @@ are always available. They are listed here in alphabetical order. | | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** | | | | | :func:`float` | | :func:`max` | | |func-set|_ | | | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | -| | :func:`bin` | | |func-frozenset|_ | | :func:`min` | | :func:`slice` | -| | :func:`bool` | | | | | | :func:`sorted` | -| | :func:`breakpoint` | | **G** | | **N** | | :func:`staticmethod` | -| | |func-bytearray|_ | | :func:`getattr` | | :func:`next` | | |func-str|_ | -| | |func-bytes|_ | | :func:`globals` | | | | :func:`sum` | -| | | | | | **O** | | :func:`super` | -| | **C** | | **H** | | :func:`object` | | | +| | :func:`bin` | | |func-frozenset|_ | | :func:`min` | | :func:`sentinel` | +| | :func:`bool` | | | | | | :func:`slice` | +| | :func:`breakpoint` | | **G** | | **N** | | :func:`sorted` | +| | |func-bytearray|_ | | :func:`getattr` | | :func:`next` | | :func:`staticmethod` | +| | |func-bytes|_ | | :func:`globals` | | | | |func-str|_ | +| | | | | | **O** | | :func:`sum` | +| | **C** | | **H** | | :func:`object` | | :func:`super` | | | :func:`callable` | | :func:`hasattr` | | :func:`oct` | | **T** | | | :func:`chr` | | :func:`hash` | | :func:`open` | | |func-tuple|_ | | | :func:`classmethod` | | :func:`help` | | :func:`ord` | | :func:`type` | @@ -138,6 +138,8 @@ are always available. They are listed here in alphabetical order. >>> f'{14:#b}', f'{14:b}' ('0b1110', '1110') + See also :func:`enum.bin` to represent negative values as twos-complement. + See also :func:`format` for more information. @@ -292,7 +294,9 @@ are always available. They are listed here in alphabetical order. :func:`property`. -.. function:: compile(source, filename, mode, flags=0, dont_inherit=False, optimize=-1) +.. function:: compile(source, filename, mode, flags=0, \ + dont_inherit=False, optimize=-1, \ + *, module=None) Compile the *source* into a code or AST object. Code objects can be executed by :func:`exec` or :func:`eval`. *source* can either be a normal string, a @@ -334,8 +338,12 @@ are always available. They are listed here in alphabetical order. ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false) or ``2`` (docstrings are removed too). - This function raises :exc:`SyntaxError` if the compiled source is invalid, - and :exc:`ValueError` if the source contains null bytes. + The optional argument *module* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax warnings + by module name. + + This function raises :exc:`SyntaxError` or :exc:`ValueError` if the compiled + source is invalid. If you want to parse Python code into its AST representation, see :func:`ast.parse`. @@ -371,6 +379,9 @@ are always available. They are listed here in alphabetical order. ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable support for top-level ``await``, ``async for``, and ``async with``. + .. versionadded:: 3.15 + Added the *module* parameter. + .. class:: complex(number=0, /) complex(string, /) @@ -517,7 +528,7 @@ are always available. They are listed here in alphabetical order. >>> dir() # show the names in the module namespace # doctest: +SKIP ['__builtins__', '__name__', 'struct'] >>> dir(struct) # show the names in the struct module # doctest: +SKIP - ['Struct', '__all__', '__builtins__', '__cached__', '__doc__', '__file__', + ['Struct', '__all__', '__builtins__', '__doc__', '__file__', '__initializing__', '__loader__', '__name__', '__package__', '_clearcache', 'calcsize', 'error', 'pack', 'pack_into', 'unpack', 'unpack_from'] @@ -583,7 +594,7 @@ are always available. They are listed here in alphabetical order. :param globals: The global namespace (default: ``None``). - :type globals: :class:`dict` | ``None`` + :type globals: :class:`dict` | :class:`frozendict` | ``None`` :param locals: The local namespace (default: ``None``). @@ -595,18 +606,19 @@ are always available. They are listed here in alphabetical order. .. warning:: This function executes arbitrary code. Calling it with - user-supplied input may lead to security vulnerabilities. + untrusted user-supplied input will lead to security vulnerabilities. - The *expression* argument is parsed and evaluated as a Python expression + The *source* argument is parsed and evaluated as a Python expression (technically speaking, a condition list) using the *globals* and *locals* mappings as global and local namespace. If the *globals* dictionary is present and does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module :mod:`builtins` is - inserted under that key before *expression* is parsed. That way you can - control what builtins are available to the executed code by inserting your - own ``__builtins__`` dictionary into *globals* before passing it to - :func:`eval`. If the *locals* mapping is omitted it defaults to the - *globals* dictionary. If both mappings are omitted, the expression is + inserted under that key before *source* is parsed. + Overriding ``__builtins__`` can be used to restrict or change the available + names, but this is **not** a security mechanism: the executed code can + still access all builtins. + If the *locals* mapping is omitted it defaults to the + *globals* dictionary. If both mappings are omitted, the source is executed with the *globals* and *locals* in the environment where :func:`eval` is called. Note, *eval()* will only have access to the :term:`nested scopes ` (non-locals) in the enclosing @@ -632,7 +644,7 @@ are always available. They are listed here in alphabetical order. If the given source is a string, then leading and trailing spaces and tabs are stripped. - See :func:`ast.literal_eval` for a function that can safely evaluate strings + See :func:`ast.literal_eval` for a function to evaluate strings with expressions containing only literals. .. audit-event:: exec code_object eval @@ -649,6 +661,10 @@ are always available. They are listed here in alphabetical order. The semantics of the default *locals* namespace have been adjusted as described for the :func:`locals` builtin. + .. versionchanged:: 3.15 + + *globals* can now be a :class:`frozendict`. + .. index:: pair: built-in function; exec .. function:: exec(source, /, globals=None, locals=None, *, closure=None) @@ -656,7 +672,7 @@ are always available. They are listed here in alphabetical order. .. warning:: This function executes arbitrary code. Calling it with - user-supplied input may lead to security vulnerabilities. + untrusted user-supplied input will lead to security vulnerabilities. This function supports dynamic execution of Python code. *source* must be either a string or a code object. If it is a string, the string is parsed as @@ -687,9 +703,10 @@ are always available. They are listed here in alphabetical order. If the *globals* dictionary does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module - :mod:`builtins` is inserted under that key. That way you can control what - builtins are available to the executed code by inserting your own - ``__builtins__`` dictionary into *globals* before passing it to :func:`exec`. + :mod:`builtins` is inserted under that key. + Overriding ``__builtins__`` can be used to restrict or change the available + names, but this is **not** a security mechanism: the executed code can + still access all builtins. The *closure* argument specifies a closure--a tuple of cellvars. It's only valid when the *object* is a code object containing @@ -726,6 +743,10 @@ are always available. They are listed here in alphabetical order. The semantics of the default *locals* namespace have been adjusted as described for the :func:`locals` builtin. + .. versionchanged:: 3.15 + + *globals* can now be a :class:`frozendict`. + .. function:: filter(function, iterable, /) @@ -1733,7 +1754,7 @@ are always available. They are listed here in alphabetical order. self.age = age def __repr__(self): - return f"Person('{self.name}', {self.age})" + return f"Person({self.name!r}, {self.age!r})" .. function:: reversed(object, /) @@ -1806,6 +1827,63 @@ are always available. They are listed here in alphabetical order. :func:`setattr`. +.. class:: sentinel(name, /) + + Return a new unique sentinel object. *name* must be a :class:`str`, and is + used as the returned object's representation:: + + >>> MISSING = sentinel("MISSING") + >>> MISSING + MISSING + + Sentinel objects are truthy and compare equal only to themselves. They are + intended to be compared with the :keyword:`is` operator. + + ``sentinel`` does not support subclassing. + + Shallow and deep copies of a sentinel object return the object itself. + + Sentinels are conventionally assigned to a variable with a matching name. + Sentinels defined in this way can be used in :term:`type hints `:: + + MISSING = sentinel("MISSING") + + def next_value(default: int | MISSING = MISSING): + ... + + Sentinel objects support the :ref:`| ` operator for use in type expressions. + + :mod:`Pickling ` is supported for sentinel objects that are + placed in the global scope of a module under a name matching the sentinel's + name, and for sentinels placed in class scopes with a name matching the + :term:`qualified name` of the sentinel. Other sentinels, such as those + defined in a function scope, are not picklable. The identity of the sentinel is preserved + after pickling:: + + import pickle + + PICKLABLE = sentinel("PICKLABLE") + + assert pickle.loads(pickle.dumps(PICKLABLE)) is PICKLABLE + + class Cls: + PICKLABLE = sentinel("Cls.PICKLABLE") + + assert pickle.loads(pickle.dumps(Cls.PICKLABLE)) is Cls.PICKLABLE + + Sentinel objects have the following attributes: + + .. attribute:: __name__ + + The sentinel's name. + + .. attribute:: __module__ + + The name of the module where the sentinel was created. + + .. versionadded:: next + + .. class:: slice(stop, /) slice(start, stop, step=None, /) @@ -1813,19 +1891,19 @@ are always available. They are listed here in alphabetical order. ``range(start, stop, step)``. The *start* and *step* arguments default to ``None``. - Slice objects have read-only data attributes :attr:`!start`, - :attr:`!stop`, and :attr:`!step` which merely return the argument - values (or their default). They have no other explicit functionality; - however, they are used by NumPy and other third-party packages. + Slice objects are also generated when :ref:`slicing syntax ` + is used. For example: ``a[start:stop:step]`` or ``a[start:stop, i]``. + + See :func:`itertools.islice` for an alternate version that returns an + :term:`iterator`. .. attribute:: slice.start - .. attribute:: slice.stop - .. attribute:: slice.step + slice.stop + slice.step - Slice objects are also generated when extended indexing syntax is used. For - example: ``a[start:stop:step]`` or ``a[start:stop, i]``. See - :func:`itertools.islice` for an alternate version that returns an - :term:`iterator`. + These read-only attributes are set to the argument values + (or their default). They have no other explicit functionality; + however, they are used by NumPy and other third-party packages. .. versionchanged:: 3.12 Slice objects are now :term:`hashable` (provided :attr:`~slice.start`, @@ -1859,7 +1937,7 @@ are always available. They are listed here in alphabetical order. the same data with other ordering tools such as :func:`max` that rely on a different underlying method. Implementing all six comparisons also helps avoid confusion for mixed type comparisons which can call - reflected the :meth:`~object.__gt__` method. + the reflected :meth:`~object.__gt__` method. For sorting examples and a brief sorting tutorial, see :ref:`sortinghowto`. @@ -2080,6 +2158,10 @@ are always available. They are listed here in alphabetical order. Subclasses of :class:`!type` which don't override ``type.__new__`` may no longer use the one-argument form to get the type of an object. + .. versionchanged:: 3.15 + + *dict* can now be a :class:`frozendict`. + .. function:: vars() vars(object, /) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index beec9b942af..7da59cba517 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -4,13 +4,6 @@ .. module:: functools :synopsis: Higher-order functions and operations on callable objects. -.. moduleauthor:: Peter Harris -.. moduleauthor:: Raymond Hettinger -.. moduleauthor:: Nick Coghlan -.. moduleauthor:: Łukasz Langa -.. moduleauthor:: Pablo Galindo -.. sectionauthor:: Peter Harris - **Source code:** :source:`Lib/functools.py` .. testsetup:: default @@ -20,11 +13,11 @@ -------------- -The :mod:`functools` module is for higher-order functions: functions that act on +The :mod:`!functools` module is for higher-order functions: functions that act on or return other functions. In general, any callable object can be treated as a function for the purposes of this module. -The :mod:`functools` module defines the following functions: +The :mod:`!functools` module defines the following functions: .. decorator:: cache(user_function) @@ -42,11 +35,11 @@ The :mod:`functools` module defines the following functions: def factorial(n): return n * factorial(n-1) if n else 1 - >>> factorial(10) # no previously cached result, makes 11 recursive calls + >>> factorial(10) # no previously cached result, makes 11 recursive calls 3628800 - >>> factorial(5) # just looks up cached value result + >>> factorial(5) # no new calls, just returns the cached result 120 - >>> factorial(12) # makes two new recursive calls, the other 10 are cached + >>> factorial(12) # two new recursive calls, factorial(10) is cached 479001600 The cache is threadsafe so that the wrapped function can be used in @@ -57,6 +50,10 @@ The :mod:`functools` module defines the following functions: another thread makes an additional call before the initial call has been completed and cached. + Call-once behavior is not guaranteed because locks are not held during the + function call. Potentially another call with the same arguments could + occur while the first call is still running. + .. versionadded:: 3.9 @@ -176,8 +173,8 @@ The :mod:`functools` module defines the following functions: the *maxsize* at its default value of 128:: @lru_cache - def count_vowels(sentence): - return sum(sentence.count(vowel) for vowel in 'AEIOUaeiou') + def count_vowels(word): + return sum(word.count(vowel) for vowel in 'AEIOUaeiou') If *maxsize* is set to ``None``, the LRU feature is disabled and the cache can grow without bound. @@ -190,7 +187,7 @@ The :mod:`functools` module defines the following functions: Note, type specificity applies only to the function's immediate arguments rather than their contents. The scalar arguments, ``Decimal(42)`` and - ``Fraction(42)`` are be treated as distinct calls with distinct results. + ``Fraction(42)`` are treated as distinct calls with distinct results. In contrast, the tuple arguments ``('answer', Decimal(42))`` and ``('answer', Fraction(42))`` are treated as equivalent. @@ -471,7 +468,7 @@ The :mod:`functools` module defines the following functions: Roughly equivalent to:: - initial_missing = object() + initial_missing = sentinel('initial_missing') def reduce(function, iterable, /, initial=initial_missing): it = iter(iterable) @@ -672,7 +669,7 @@ The :mod:`functools` module defines the following functions: dispatch>` :term:`generic function`. To define a generic method, decorate it with the ``@singledispatchmethod`` - decorator. When defining a function using ``@singledispatchmethod``, note + decorator. When defining a method using ``@singledispatchmethod``, note that the dispatch happens on the type of the first non-*self* or non-*cls* argument:: @@ -690,7 +687,7 @@ The :mod:`functools` module defines the following functions: return not arg ``@singledispatchmethod`` supports nesting with other decorators such as - :func:`@classmethod`. Note that to allow for + :deco:`classmethod`. Note that to allow for ``dispatcher.register``, ``singledispatchmethod`` must be the *outer most* decorator. Here is the ``Negator`` class with the ``neg`` methods bound to the class, rather than an instance of the class:: @@ -712,11 +709,13 @@ The :mod:`functools` module defines the following functions: return not arg The same pattern can be used for other similar decorators: - :func:`@staticmethod`, - :func:`@abstractmethod`, and others. + :deco:`staticmethod`, :deco:`~abc.abstractmethod`, and others. .. versionadded:: 3.8 + .. versionchanged:: 3.15 + Added support of non-:term:`descriptor` callables. + .. function:: update_wrapper(wrapper, wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 2ef5c4b35a2..701af579453 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -4,9 +4,6 @@ .. module:: gc :synopsis: Interface to the cycle-detecting garbage collector. -.. moduleauthor:: Neil Schemenauer -.. sectionauthor:: Neil Schemenauer - -------------- This module provides an interface to the optional garbage collector. It @@ -20,7 +17,7 @@ can be disabled by calling ``gc.disable()``. To debug a leaking program call ``gc.DEBUG_SAVEALL``, causing garbage-collected objects to be saved in gc.garbage for inspection. -The :mod:`gc` module provides the following functions: +The :mod:`!gc` module provides the following functions: .. function:: enable() @@ -40,18 +37,11 @@ The :mod:`gc` module provides the following functions: .. function:: collect(generation=2) - Perform a collection. The optional argument *generation* + With no arguments, run a full collection. The optional argument *generation* may be an integer specifying which generation to collect (from 0 to 2). A :exc:`ValueError` is raised if the generation number is invalid. The sum of collected objects and uncollectable objects is returned. - Calling ``gc.collect(0)`` will perform a GC collection on the young generation. - - Calling ``gc.collect(1)`` will perform a GC collection on the young generation - and an increment of the old generation. - - Calling ``gc.collect(2)`` or ``gc.collect()`` performs a full collection - The free lists maintained for a number of built-in types are cleared whenever a full collection or collection of the highest generation (2) is run. Not all items in some free lists may be freed due to the @@ -63,6 +53,9 @@ The :mod:`gc` module provides the following functions: .. versionchanged:: 3.14 ``generation=1`` performs an increment of collection. + .. versionchanged:: 3.14.5 + ``generation=1`` performs collection of the middle generation. + .. function:: set_debug(flags) @@ -78,13 +71,9 @@ The :mod:`gc` module provides the following functions: .. function:: get_objects(generation=None) - Returns a list of all objects tracked by the collector, excluding the list - returned. If *generation* is not ``None``, return only the objects as follows: - - * 0: All objects in the young generation - * 1: No objects, as there is no generation 1 (as of Python 3.14) - * 2: All objects in the old generation + returned. If *generation* is not ``None``, return only the objects tracked by + the collector that are in that generation. .. versionchanged:: 3.8 New *generation* parameter. @@ -92,6 +81,9 @@ The :mod:`gc` module provides the following functions: .. versionchanged:: 3.14 Generation 1 is removed + .. versionchanged:: 3.14.5 + Generation 1 is reintroduced to maintain GC behavior from 3.13. + .. audit-event:: gc.get_objects generation gc.get_objects .. function:: get_stats() @@ -108,43 +100,52 @@ The :mod:`gc` module provides the following functions: * ``uncollectable`` is the total number of objects which were found to be uncollectable (and were therefore moved to the :data:`garbage` - list) inside this generation. + list) inside this generation; + + * ``candidates`` is the total number of objects in this generation which were + considered for collection and traversed; + + * ``duration`` is the total time in seconds spent in collections for this + generation. .. versionadded:: 3.4 + .. versionchanged:: 3.15 + Add ``duration`` and ``candidates``. + .. function:: set_threshold(threshold0, [threshold1, [threshold2]]) Set the garbage collection thresholds (the collection frequency). Setting *threshold0* to zero disables collection. - The GC classifies objects into two generations depending on whether they have - survived a collection. New objects are placed in the young generation. If an - object survives a collection it is moved into the old generation. - - In order to decide when to run, the collector keeps track of the number of object + The GC classifies objects into three generations depending on how many + collection sweeps they have survived. New objects are placed in the youngest + generation (generation ``0``). If an object survives a collection it is moved + into the next older generation. Since generation ``2`` is the oldest + generation, objects in that generation remain there after a collection. In + order to decide when to run, the collector keeps track of the number object allocations and deallocations since the last collection. When the number of allocations minus the number of deallocations exceeds *threshold0*, collection - starts. For each collection, all the objects in the young generation and some - fraction of the old generation is collected. + starts. Initially only generation ``0`` is examined. If generation ``0`` has + been examined more than *threshold1* times since generation ``1`` has been + examined, then generation ``1`` is examined as well. + With the third generation, things are a bit more complicated, + see `Collecting the oldest generation `_ for more information. In the free-threaded build, the increase in process memory usage is also checked before running the collector. If the memory usage has not increased by 10% since the last collection and the net number of object allocations has not exceeded 40 times *threshold0*, the collection is not run. - The fraction of the old generation that is collected is **inversely** proportional - to *threshold1*. The larger *threshold1* is, the slower objects in the old generation - are collected. - For the default value of 10, 1% of the old generation is scanned during each collection. - - *threshold2* is ignored. - - See `Garbage collector design `_ for more information. + See `Garbage collector design `_ for more information. .. versionchanged:: 3.14 *threshold2* is ignored + .. versionchanged:: 3.14.5 + *threshold2* is restored to match Python 3.13 behavior. + .. function:: get_count() @@ -313,6 +314,12 @@ values but should not rebind them): "uncollectable": When *phase* is "stop", the number of objects that could not be collected and were put in :data:`garbage`. + "candidates": When *phase* is "stop", the total number of objects in this + generation which were considered for collection and traversed. + + "duration": When *phase* is "stop", the time in seconds spent in the + collection. + Applications can add their own callbacks to this list. The primary use cases are: @@ -325,6 +332,9 @@ values but should not rebind them): .. versionadded:: 3.3 + .. versionchanged:: 3.15 + Add "duration" and "candidates". + The following constants are provided for use with :func:`set_debug`: diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst index af9c9e9f39d..fd96f3bbf6a 100644 --- a/Doc/library/getpass.rst +++ b/Doc/library/getpass.rst @@ -4,17 +4,13 @@ .. module:: getpass :synopsis: Portable reading of passwords and retrieval of the userid. -.. moduleauthor:: Piers Lauder -.. sectionauthor:: Fred L. Drake, Jr. -.. Windows (& Mac?) support by Guido van Rossum. - **Source code:** :source:`Lib/getpass.py` -------------- .. include:: ../includes/wasm-notavail.rst -The :mod:`getpass` module provides two functions: +The :mod:`!getpass` module provides two functions: .. function:: getpass(prompt='Password: ', stream=None, *, echo_char=None) @@ -27,9 +23,9 @@ The :mod:`getpass` module provides two functions: The *echo_char* argument controls how user input is displayed while typing. If *echo_char* is ``None`` (default), input remains hidden. Otherwise, - *echo_char* must be a printable ASCII string and each typed character - is replaced by it. For example, ``echo_char='*'`` will display - asterisks instead of the actual input. + *echo_char* must be a single printable ASCII character and each + typed character is replaced by it. For example, ``echo_char='*'`` will + display asterisks instead of the actual input. If echo free input is unavailable getpass() falls back to printing a warning message to *stream* and reading from ``sys.stdin`` and @@ -43,13 +39,27 @@ The :mod:`getpass` module provides two functions: On Unix systems, when *echo_char* is set, the terminal will be configured to operate in :manpage:`noncanonical mode `. - In particular, this means that line editing shortcuts such as - :kbd:`Ctrl+U` will not work and may insert unexpected characters into - the input. + Common terminal control characters are supported: + + * :kbd:`Ctrl+A` - Move cursor to beginning of line + * :kbd:`Ctrl+E` - Move cursor to end of line + * :kbd:`Ctrl+K` - Kill (delete) from cursor to end of line + * :kbd:`Ctrl+U` - Kill (delete) entire line + * :kbd:`Ctrl+W` - Erase previous word + * :kbd:`Ctrl+V` - Insert next character literally (quote) + * :kbd:`Backspace`/:kbd:`DEL` - Delete character before cursor + + These shortcuts work by reading the terminal's configured control + character mappings from termios settings. .. versionchanged:: 3.14 Added the *echo_char* parameter for keyboard feedback. + .. versionchanged:: 3.15 + When using non-empty *echo_char* on Unix, keyboard shortcuts (including + cursor movement and line editing) are now properly handled using the + terminal's control character configuration. + .. exception:: GetPassWarning A :exc:`UserWarning` subclass issued when password input may be echoed. diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index d0de83907eb..2de16fe4036 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -4,14 +4,11 @@ .. module:: gettext :synopsis: Multilingual internationalization services. -.. moduleauthor:: Barry A. Warsaw -.. sectionauthor:: Barry A. Warsaw - **Source code:** :source:`Lib/gettext.py` -------------- -The :mod:`gettext` module provides internationalization (I18N) and localization +The :mod:`!gettext` module provides internationalization (I18N) and localization (L10N) services for your Python modules and applications. It supports both the GNU :program:`gettext` message catalog API and a higher level, class-based API that may be more appropriate for Python files. The interface described below allows you @@ -25,7 +22,7 @@ Some hints on localizing your Python modules and applications are also given. GNU :program:`gettext` API -------------------------- -The :mod:`gettext` module defines the following API, which is very similar to +The :mod:`!gettext` module defines the following API, which is very similar to the GNU :program:`gettext` API. If you use this API you will affect the translation of your entire application globally. Often this is what you want if your application is monolingual, with the choice of language dependent on the @@ -37,7 +34,7 @@ class-based API instead. .. function:: bindtextdomain(domain, localedir=None) Bind the *domain* to the locale directory *localedir*. More concretely, - :mod:`gettext` will look for binary :file:`.mo` files for the given domain using + :mod:`!gettext` will look for binary :file:`.mo` files for the given domain using the path (on Unix): :file:`{localedir}/{language}/LC_MESSAGES/{domain}.mo`, where *language* is searched for in the environment variables :envvar:`LANGUAGE`, :envvar:`LC_ALL`, :envvar:`LC_MESSAGES`, and :envvar:`LANG` respectively. @@ -114,7 +111,7 @@ Here's an example of typical usage for this API:: Class-based API --------------- -The class-based API of the :mod:`gettext` module gives you more flexibility and +The class-based API of the :mod:`!gettext` module gives you more flexibility and greater convenience than the GNU :program:`gettext` API. It is the recommended way of localizing your Python applications and modules. :mod:`!gettext` defines a :class:`GNUTranslations` class which implements the parsing of GNU :file:`.mo` format @@ -393,7 +390,7 @@ The Catalog constructor .. index:: single: GNOME -GNOME uses a version of the :mod:`gettext` module by James Henstridge, but this +GNOME uses a version of the :mod:`!gettext` module by James Henstridge, but this version has a slightly different API. Its documented usage was:: import gettext @@ -425,7 +422,7 @@ take the following steps: #. create language-specific translations of the message catalogs -#. use the :mod:`gettext` module so that message strings are properly translated +#. use the :mod:`!gettext` module so that message strings are properly translated In order to prepare your code for I18N, you need to look at all the strings in your files. Any string that needs to be translated should be marked by wrapping @@ -473,10 +470,10 @@ supported natural language. They send back the completed language-specific versions as a :file:`.po` file that's compiled into a machine-readable :file:`.mo` binary catalog file using the :program:`msgfmt` program. The :file:`.mo` files are used by the -:mod:`gettext` module for the actual translation processing at +:mod:`!gettext` module for the actual translation processing at run-time. -How you use the :mod:`gettext` module in your code depends on whether you are +How you use the :mod:`!gettext` module in your code depends on whether you are internationalizing a single module or your entire application. The next two sections will discuss each case. diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 59ad1b07f27..942f23d216f 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -18,23 +18,27 @@ single: - (minus); in glob-style wildcards single: . (dot); in glob-style wildcards -The :mod:`glob` module finds all the pathnames matching a specified pattern -according to the rules used by the Unix shell, although results are returned in -arbitrary order. No tilde expansion is done, but ``*``, ``?``, and character +The :mod:`!glob` module finds pathnames +using pattern matching rules similar to the Unix shell. +No tilde expansion is done, but ``*``, ``?``, and character ranges expressed with ``[]`` will be correctly matched. This is done by using the :func:`os.scandir` and :func:`fnmatch.fnmatch` functions in concert, and not by actually invoking a subshell. -Note that files beginning with a dot (``.``) can only be matched by +.. note:: + The pathnames are returned in no particular order. If you need a specific + order, sort the results. + +Files beginning with a dot (``.``) can only be matched by patterns that also start with a dot, unlike :func:`fnmatch.fnmatch` or :func:`pathlib.Path.glob`. -(For tilde and shell variable expansion, use :func:`os.path.expanduser` and -:func:`os.path.expandvars`.) +For tilde and shell variable expansion, use :func:`os.path.expanduser` and +:func:`os.path.expandvars`. For a literal match, wrap the meta-characters in brackets. For example, ``'[?]'`` matches the character ``'?'``. -The :mod:`glob` module defines the following functions: +The :mod:`!glob` module defines the following functions: .. function:: glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, \ @@ -51,7 +55,7 @@ The :mod:`glob` module defines the following functions: If *root_dir* is not ``None``, it should be a :term:`path-like object` specifying the root directory for searching. It has the same effect on - :func:`glob` as changing the current directory before calling it. If + :func:`!glob` as changing the current directory before calling it. If *pathname* is relative, the result will contain paths relative to *root_dir*. @@ -79,6 +83,11 @@ The :mod:`glob` module defines the following functions: This function may return duplicate path names if *pathname* contains multiple "``**``" patterns and *recursive* is true. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". @@ -102,6 +111,11 @@ The :mod:`glob` module defines the following functions: This function may return duplicate path names if *pathname* contains multiple "``**``" patterns and *recursive* is true. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". @@ -126,7 +140,8 @@ The :mod:`glob` module defines the following functions: .. function:: translate(pathname, *, recursive=False, include_hidden=False, seps=None) Convert the given path specification to a regular expression for use with - :func:`re.match`. The path specification can contain shell-style wildcards. + :func:`re.prefixmatch`. The path specification can contain shell-style + wildcards. For example: @@ -136,7 +151,7 @@ The :mod:`glob` module defines the following functions: >>> regex '(?s:(?:.+/)?[^/]*\\.txt)\\z' >>> reobj = re.compile(regex) - >>> reobj.match('foo/bar/baz.txt') + >>> reobj.prefixmatch('foo/bar/baz.txt') Path separators and segments are meaningful to this function, unlike diff --git a/Doc/library/graphlib.rst b/Doc/library/graphlib.rst index 053d5f8231b..21f4d1fb938 100644 --- a/Doc/library/graphlib.rst +++ b/Doc/library/graphlib.rst @@ -204,7 +204,7 @@ Exceptions ---------- -The :mod:`graphlib` module defines the following exception classes: +The :mod:`!graphlib` module defines the following exception classes: .. exception:: CycleError diff --git a/Doc/library/grp.rst b/Doc/library/grp.rst index d1c7f22a209..f436970e791 100644 --- a/Doc/library/grp.rst +++ b/Doc/library/grp.rst @@ -2,7 +2,6 @@ ================================== .. module:: grp - :platform: Unix :synopsis: The group database (getgrnam() and friends). -------------- diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index c59014a6f5b..ed9fdaf1d72 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -11,9 +11,11 @@ This module provides a simple interface to compress and decompress files just like the GNU programs :program:`gzip` and :program:`gunzip` would. +.. include:: ../includes/optional-module.rst + The data compression is provided by the :mod:`zlib` module. -The :mod:`gzip` module provides the :class:`GzipFile` class, as well as the +The :mod:`!gzip` module provides the :class:`GzipFile` class, as well as the :func:`.open`, :func:`compress` and :func:`decompress` convenience functions. The :class:`GzipFile` class reads and writes :program:`gzip`\ -format files, automatically compressing or decompressing the data so that it looks like an @@ -59,7 +61,7 @@ The module defines the following items: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. versionchanged:: next + .. versionchanged:: 3.15 The default compression level was reduced to 6 (down from 9). It is the default level used by most compression tools and a better tradeoff between speed and performance. @@ -186,7 +188,7 @@ The module defines the following items: Remove the ``filename`` attribute, use the :attr:`~GzipFile.name` attribute instead. - .. versionchanged:: next + .. versionchanged:: 3.15 The default compression level was reduced to 6 (down from 9). It is the default level used by most compression tools and a better tradeoff between speed and performance. @@ -216,7 +218,7 @@ The module defines the following items: The *mtime* parameter now defaults to 0 for reproducible output. For the previous behaviour of using the current time, pass ``None`` to *mtime*. - .. versionchanged:: next + .. versionchanged:: 3.15 The default compression level was reduced to 6 (down from 9). It is the default level used by most compression tools and a better tradeoff between speed and performance. @@ -281,20 +283,20 @@ Example of how to GZIP compress a binary string:: .. _gzip-cli: -Command Line Interface +Command-line interface ---------------------- -The :mod:`gzip` module provides a simple command line interface to compress or +The :mod:`!gzip` module provides a simple command line interface to compress or decompress files. -Once executed the :mod:`gzip` module keeps the input file(s). +Once executed the :mod:`!gzip` module keeps the input file(s). .. versionchanged:: 3.8 Add a new command line interface with a usage. By default, when you will execute the CLI, the default compression level is 6. -Command line options +Command-line options ^^^^^^^^^^^^^^^^^^^^ .. option:: file diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 8bba6700930..ed0b0b2735b 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -4,9 +4,6 @@ .. module:: hashlib :synopsis: Secure hash and message digest algorithms. -.. moduleauthor:: Gregory P. Smith -.. sectionauthor:: Gregory P. Smith - **Source code:** :source:`Lib/hashlib.py` .. index:: @@ -61,7 +58,7 @@ if you are using a rare "FIPS compliant" build of Python. These correspond to :data:`algorithms_guaranteed`. Additional algorithms may also be available if your Python distribution's -:mod:`hashlib` was linked against a build of OpenSSL that provides others. +:mod:`!hashlib` was linked against a build of OpenSSL that provides others. Others *are not guaranteed available* on all installations and will only be accessible by name via :func:`new`. See :data:`algorithms_available`. @@ -310,7 +307,7 @@ a file or file-like object. .. versionadded:: 3.11 .. versionchanged:: 3.14 - Now raises a :exc:`BlockingIOError` if the file is opened in blocking + Now raises a :exc:`BlockingIOError` if the file is opened in non-blocking mode. Previously, spurious null bytes were added to the digest. @@ -379,8 +376,6 @@ include a `salt `_. BLAKE2 ------ -.. sectionauthor:: Dmitry Chestnykh - .. index:: single: blake2b, blake2s @@ -397,7 +392,7 @@ BLAKE2 supports **keyed mode** (a faster and simpler replacement for HMAC_), **salted hashing**, **personalization**, and **tree hashing**. Hash objects from this module follow the API of standard library's -:mod:`hashlib` objects. +:mod:`!hashlib` objects. Creating hash objects diff --git a/Doc/library/heapq.rst b/Doc/library/heapq.rst index 95ef72469b1..26cffa7c643 100644 --- a/Doc/library/heapq.rst +++ b/Doc/library/heapq.rst @@ -4,11 +4,6 @@ .. module:: heapq :synopsis: Heap queue algorithm (a.k.a. priority queue). -.. moduleauthor:: Kevin O'Connor -.. sectionauthor:: Guido van Rossum -.. sectionauthor:: François Pinard -.. sectionauthor:: Raymond Hettinger - **Source code:** :source:`Lib/heapq.py` -------------- @@ -58,6 +53,11 @@ functions, respectively. The following functions are provided for min-heaps: +.. function:: heapify(x) + + Transform list *x* into a min-heap, in-place, in linear time. + + .. function:: heappush(heap, item) Push the value *item* onto the *heap*, maintaining the min-heap invariant. @@ -77,11 +77,6 @@ The following functions are provided for min-heaps: followed by a separate call to :func:`heappop`. -.. function:: heapify(x) - - Transform list *x* into a min-heap, in-place, in linear time. - - .. function:: heapreplace(heap, item) Pop and return the smallest item from the *heap*, and also push the new *item*. diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index d5608bd7543..2ee0c0bd912 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -4,9 +4,6 @@ .. module:: hmac :synopsis: Keyed-Hashing for Message Authentication (HMAC) implementation -.. moduleauthor:: Gerhard Häring -.. sectionauthor:: Gerhard Häring - **Source code:** :source:`Lib/hmac.py` -------------- diff --git a/Doc/library/html.entities.rst b/Doc/library/html.entities.rst index add18e4c87d..15d2dc2e9aa 100644 --- a/Doc/library/html.entities.rst +++ b/Doc/library/html.entities.rst @@ -4,8 +4,6 @@ .. module:: html.entities :synopsis: Definitions of HTML general entities. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/html/entities.py` -------------- diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst index dd67fc34e85..11f851d4f6c 100644 --- a/Doc/library/html.parser.rst +++ b/Doc/library/html.parser.rst @@ -15,14 +15,18 @@ This module defines a class :class:`HTMLParser` which serves as the basis for parsing text files formatted in HTML (HyperText Mark-up Language) and XHTML. -.. class:: HTMLParser(*, convert_charrefs=True) +.. class:: HTMLParser(*, convert_charrefs=True, scripting=False) Create a parser instance able to parse invalid markup. - If *convert_charrefs* is ``True`` (the default), all character - references (except the ones in ``script``/``style`` elements) are + If *convert_charrefs* is true (the default), all character + references (except the ones in elements like ``script`` and ``style``) are automatically converted to the corresponding Unicode characters. + If *scripting* is false (the default), the content of the ``noscript`` + element is parsed normally; if it's true, it's returned as is without + being parsed. + An :class:`.HTMLParser` instance is fed HTML data and calls handler methods when start tags, end tags, text, comments, and other markup elements are encountered. The user should subclass :class:`.HTMLParser` and override its @@ -37,6 +41,9 @@ parsing text files formatted in HTML (HyperText Mark-up Language) and XHTML. .. versionchanged:: 3.5 The default value for argument *convert_charrefs* is now ``True``. + .. versionchanged:: 3.14.1 + Added the *scripting* parameter. + Example HTML Parser Application ------------------------------- @@ -134,7 +141,7 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): argument is a list of ``(name, value)`` pairs containing the attributes found inside the tag's ``<>`` brackets. The *name* will be translated to lower case, and quotes in the *value* have been removed, and character and entity references - have been replaced. + have been replaced. For empty attributes, *value* is ``None``. For instance, for the tag ````, this method would be called as ``handle_starttag('a', [('href', 'https://www.cwi.nl/')])``. @@ -161,15 +168,15 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): .. method:: HTMLParser.handle_data(data) This method is called to process arbitrary data (e.g. text nodes and the - content of ```` and ````). + content of elements like ``script`` and ``style``). .. method:: HTMLParser.handle_entityref(name) This method is called to process a named character reference of the form ``&name;`` (e.g. ``>``), where *name* is a general entity reference - (e.g. ``'gt'``). This method is never called if *convert_charrefs* is - ``True``. + (e.g. ``'gt'``). + This method is only called if *convert_charrefs* is false. .. method:: HTMLParser.handle_charref(name) @@ -177,8 +184,8 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): This method is called to process decimal and hexadecimal numeric character references of the form :samp:`&#{NNN};` and :samp:`&#x{NNN};`. For example, the decimal equivalent for ``>`` is ``>``, whereas the hexadecimal is ``>``; - in this case the method will receive ``'62'`` or ``'x3E'``. This method - is never called if *convert_charrefs* is ``True``. + in this case the method will receive ``'62'`` or ``'x3E'``. + This method is only called if *convert_charrefs* is false. .. method:: HTMLParser.handle_comment(data) @@ -292,8 +299,8 @@ Parsing an element with a few attributes and a title: Data : Python End tag : h1 -The content of ``script`` and ``style`` elements is returned as is, without -further parsing: +The content of elements like ``script`` and ``style`` is returned as is, +without further parsing: .. doctest:: @@ -304,12 +311,24 @@ further parsing: End tag : style >>> parser.feed('') + ... 'alert("hello! ☺");') Start tag: script attr: ('type', 'text/javascript') - Data : alert("hello!"); + Data : alert("hello! ☺"); End tag : script +Attribute names are converted to lowercase, quotes from attribute values removed, +and ``None`` is returned as *value* for empty attributes (such as ``checked``): + +.. doctest:: + + >>> parser.feed("") + Start tag: input + attr: ('type', 'checkbox') + attr: ('checked', None) + attr: ('required', '') + attr: ('disabled', 'disabled') + Parsing comments: .. doctest:: @@ -336,7 +355,7 @@ correct char (note: these 3 references are all equivalent to ``'>'``): Feeding incomplete chunks to :meth:`~HTMLParser.feed` works, but :meth:`~HTMLParser.handle_data` might be called more than once -(unless *convert_charrefs* is set to ``True``): +if *convert_charrefs* is false: .. doctest:: diff --git a/Doc/library/html.rst b/Doc/library/html.rst index 9aa39ba9a42..65c49a4107a 100644 --- a/Doc/library/html.rst +++ b/Doc/library/html.rst @@ -14,9 +14,12 @@ This module defines utilities to manipulate HTML. Convert the characters ``&``, ``<`` and ``>`` in string *s* to HTML-safe sequences. Use this if you need to display text that might contain such - characters in HTML. If the optional flag *quote* is true, the characters - (``"``) and (``'``) are also translated; this helps for inclusion in an HTML - attribute value delimited by quotes, as in ````. + characters in HTML. If the optional flag *quote* is true (the default), the + characters (``"``) and (``'``) are also translated; this helps for inclusion + in an HTML attribute value delimited by quotes, as in ````. + If *quote* is set to false, the characters (``"``) and (``'``) are not + translated. + .. versionadded:: 3.2 diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 07f5ebf57c9..ddf3d40d221 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -68,7 +68,7 @@ The module provides the following classes: .. versionchanged:: 3.7 *blocksize* parameter was added. - .. versionchanged:: next + .. versionchanged:: 3.15 *max_response_headers* parameter was added. @@ -114,7 +114,7 @@ The module provides the following classes: The deprecated *key_file*, *cert_file* and *check_hostname* parameters have been removed. - .. versionchanged:: next + .. versionchanged:: 3.15 *max_response_headers* parameter was added. @@ -133,7 +133,7 @@ This module provides the following function: Parse the headers from a file pointer *fp* representing a HTTP request/response. The file has to be a :class:`~io.BufferedIOBase` reader - (i.e. not text) and must provide a valid :rfc:`2822` style header. + (i.e. not text) and must provide a valid :rfc:`5322` style header. This function returns an instance of :class:`http.client.HTTPMessage` that holds the header fields, but no payload @@ -319,6 +319,12 @@ HTTPConnection Objects :class:`str` or bytes-like object that is not also a file as the body representation. + .. note:: + + Note that you must have read the whole response or call :meth:`close` + if :meth:`getresponse` raised an non-:exc:`ConnectionError` exception + before you can send a new request to the server. + .. versionchanged:: 3.2 *body* can now be an iterable. @@ -334,16 +340,15 @@ HTTPConnection Objects Should be called after a request is sent to get the response from the server. Returns an :class:`HTTPResponse` instance. - .. note:: - - Note that you must have read the whole response before you can send a new - request to the server. - .. versionchanged:: 3.5 If a :exc:`ConnectionError` or subclass is raised, the :class:`HTTPConnection` object will be ready to reconnect when a new request is sent. + Note that this does not apply to :exc:`OSError`\s raised by the underlying + socket. Instead the caller is responsible to call :meth:`close` on the + existing connection. + .. method:: HTTPConnection.set_debuglevel(level) @@ -429,7 +434,7 @@ HTTPConnection Objects The maximum number of allowed response headers to help prevent denial-of-service attacks. By default, the maximum number of allowed headers is set to 100. - .. versionadded:: next + .. versionadded:: 3.15 As an alternative to using the :meth:`~HTTPConnection.request` method described above, you can diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index 251aea891c3..5ee783b7fae 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -4,15 +4,12 @@ .. module:: http.cookiejar :synopsis: Classes for automatic handling of HTTP cookies. -.. moduleauthor:: John J. Lee -.. sectionauthor:: John J. Lee - **Source code:** :source:`Lib/http/cookiejar.py` -------------- -The :mod:`http.cookiejar` module defines classes for automatic handling of HTTP -cookies. It is useful for accessing web sites that require small pieces of data +The :mod:`!http.cookiejar` module defines classes for automatic handling of HTTP +cookies. It is useful for accessing websites that require small pieces of data -- :dfn:`cookies` -- to be set on the client machine by an HTTP response from a web server, and then returned to the server in later HTTP requests. @@ -21,7 +18,7 @@ Both the regular Netscape cookie protocol and the protocol defined by :rfc:`2109` cookies are parsed as Netscape cookies and subsequently treated either as Netscape or RFC 2965 cookies according to the 'policy' in effect. Note that the great majority of cookies on the internet are Netscape cookies. -:mod:`http.cookiejar` attempts to follow the de-facto Netscape cookie protocol (which +:mod:`!http.cookiejar` attempts to follow the de-facto Netscape cookie protocol (which differs substantially from that set out in the original Netscape specification), including taking note of the ``max-age`` and ``port`` cookie-attributes introduced with RFC 2965. @@ -109,7 +106,7 @@ The following classes are provided: .. class:: Cookie() This class represents Netscape, :rfc:`2109` and :rfc:`2965` cookies. It is not - expected that users of :mod:`http.cookiejar` construct their own :class:`Cookie` + expected that users of :mod:`!http.cookiejar` construct their own :class:`Cookie` instances. Instead, if necessary, call :meth:`make_cookies` on a :class:`CookieJar` instance. @@ -121,13 +118,13 @@ The following classes are provided: Module :mod:`http.cookies` HTTP cookie classes, principally useful for server-side code. The - :mod:`http.cookiejar` and :mod:`http.cookies` modules do not depend on each + :mod:`!http.cookiejar` and :mod:`http.cookies` modules do not depend on each other. https://curl.se/rfc/cookie_spec.html The specification of the original Netscape cookie protocol. Though this is still the dominant protocol, the 'Netscape cookie protocol' implemented by all - the major browsers (and :mod:`http.cookiejar`) only bears a passing resemblance to + the major browsers (and :mod:`!http.cookiejar`) only bears a passing resemblance to the one sketched out in ``cookie_spec.html``. :rfc:`2109` - HTTP State Management Mechanism @@ -617,7 +614,7 @@ standard cookie-attributes specified in the various cookie standards. The correspondence is not one-to-one, because there are complicated rules for assigning default values, because the ``max-age`` and ``expires`` cookie-attributes contain equivalent information, and because :rfc:`2109` cookies -may be 'downgraded' by :mod:`http.cookiejar` from version 1 to version 0 (Netscape) +may be 'downgraded' by :mod:`!http.cookiejar` from version 1 to version 0 (Netscape) cookies. Assignment to these attributes should not be necessary other than in rare @@ -629,7 +626,7 @@ internal consistency, so you should know what you're doing if you do that. Integer or :const:`None`. Netscape cookies have :attr:`version` 0. :rfc:`2965` and :rfc:`2109` cookies have a ``version`` cookie-attribute of 1. However, note that - :mod:`http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in which + :mod:`!http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in which case :attr:`version` is 0. @@ -692,7 +689,7 @@ internal consistency, so you should know what you're doing if you do that. ``True`` if this cookie was received as an :rfc:`2109` cookie (ie. the cookie arrived in a :mailheader:`Set-Cookie` header, and the value of the Version cookie-attribute in that header was 1). This attribute is provided because - :mod:`http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in + :mod:`!http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in which case :attr:`version` is 0. @@ -744,7 +741,7 @@ The :class:`Cookie` class also defines the following method: Examples -------- -The first example shows the most common usage of :mod:`http.cookiejar`:: +The first example shows the most common usage of :mod:`!http.cookiejar`:: import http.cookiejar, urllib.request cj = http.cookiejar.CookieJar() diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index 9e7648ef6d8..1122b30d29d 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -4,14 +4,11 @@ .. module:: http.cookies :synopsis: Support for HTTP state management (cookies). -.. moduleauthor:: Timothy O'Malley -.. sectionauthor:: Moshe Zadka - **Source code:** :source:`Lib/http/cookies.py` -------------- -The :mod:`http.cookies` module defines classes for abstracting the concept of +The :mod:`!http.cookies` module defines classes for abstracting the concept of cookies, an HTTP state management mechanism. It supports both simple string-only cookies, and provides an abstraction for having any serializable data-type as cookie value. @@ -28,10 +25,8 @@ The character set, :data:`string.ascii_letters`, :data:`string.digits` and in a cookie name (as :attr:`~Morsel.key`). .. versionchanged:: 3.3 - Allowed '``:``' as a valid cookie name character. + Allowed ':' as a valid cookie name character. -.. versionchanged:: next - Allowed '``"``' as a valid cookie value character. .. note:: @@ -67,7 +62,7 @@ in a cookie name (as :attr:`~Morsel.key`). Module :mod:`http.cookiejar` HTTP cookie handling for web *clients*. The :mod:`http.cookiejar` and - :mod:`http.cookies` modules do not depend on each other. + :mod:`!http.cookies` modules do not depend on each other. :rfc:`2109` - HTTP State Management Mechanism This is the state management specification implemented by this module. @@ -266,7 +261,7 @@ Morsel Objects Example ------- -The following example demonstrates how to use the :mod:`http.cookies` module. +The following example demonstrates how to use the :mod:`!http.cookies` module. .. doctest:: :options: +NORMALIZE_WHITESPACE @@ -294,9 +289,9 @@ The following example demonstrates how to use the :mod:`http.cookies` module. Set-Cookie: chips=ahoy Set-Cookie: vienna=finger >>> C = cookies.SimpleCookie() - >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=\\012;";') + >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=;";') >>> print(C) - Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=\012;" + Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=;" >>> C = cookies.SimpleCookie() >>> C["oreo"] = "doublestuff" >>> C["oreo"]["path"] = "/" @@ -316,10 +311,3 @@ The following example demonstrates how to use the :mod:`http.cookies` module. >>> print(C) Set-Cookie: number=7 Set-Cookie: string=seven - >>> import json - >>> C = cookies.SimpleCookie() - >>> C.load(f'cookies=7; mixins="{json.dumps({"chips": "dark chocolate"})}"; state=gooey') - >>> print(C) - Set-Cookie: cookies=7 - Set-Cookie: mixins="{"chips": "dark chocolate"}" - Set-Cookie: state=gooey diff --git a/Doc/library/http.rst b/Doc/library/http.rst index ce3fb9f8120..43a801416e2 100644 --- a/Doc/library/http.rst +++ b/Doc/library/http.rst @@ -12,7 +12,7 @@ -------------- -:mod:`http` is a package that collects several modules for working with the +:mod:`!http` is a package that collects several modules for working with the HyperText Transfer Protocol: * :mod:`http.client` is a low-level HTTP protocol client; for high-level URL @@ -22,7 +22,7 @@ HyperText Transfer Protocol: * :mod:`http.cookiejar` provides persistence of cookies -The :mod:`http` module also defines the following enums that help you work with http related code: +The :mod:`!http` module also defines the following enums that help you work with http related code: .. class:: HTTPStatus @@ -139,7 +139,8 @@ equal to the constant name (i.e. ``http.HTTPStatus.OK`` is also available as .. versionchanged:: 3.13 Implemented RFC9110 naming for status constants. Old constant names are preserved for - backwards compatibility. + backwards compatibility: ``413 REQUEST_ENTITY_TOO_LARGE``, ``414 REQUEST_URI_TOO_LONG``, + ``416 REQUESTED_RANGE_NOT_SATISFIABLE`` and ``422 UNPROCESSABLE_ENTITY``. HTTP status category -------------------- diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 063344e0284..5f325df5570 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -19,7 +19,7 @@ This module defines classes for implementing HTTP servers. .. warning:: - :mod:`http.server` is not recommended for production. It only implements + :mod:`!http.server` is not recommended for production. It only implements :ref:`basic security checks `. .. include:: ../includes/wasm-notavail.rst @@ -99,7 +99,7 @@ instantiation, of which this module provides three different variants: This class is used to handle the HTTP requests that arrive at the server. By itself, it cannot respond to any actual HTTP requests; it must be subclassed - to handle each request method (e.g. GET or POST). + to handle each request method (for example, ``'GET'`` or ``'POST'``). :class:`BaseHTTPRequestHandler` provides a number of class and instance variables, and methods for use by subclasses. @@ -154,7 +154,7 @@ instantiation, of which this module provides three different variants: variable. This instance parses and manages the headers in the HTTP request. The :func:`~http.client.parse_headers` function from :mod:`http.client` is used to parse the headers and it requires that the - HTTP request provide a valid :rfc:`2822` style header. + HTTP request provide a valid :rfc:`5322` style header. .. attribute:: rfile @@ -241,7 +241,7 @@ instantiation, of which this module provides three different variants: request header it responds back with a ``100 Continue`` followed by ``200 OK`` headers. This method can be overridden to raise an error if the server does not - want the client to continue. For e.g. server can choose to send ``417 + want the client to continue. For example, the server can choose to send ``417 Expectation Failed`` as a response header and ``return False``. .. versionadded:: 3.2 @@ -287,6 +287,8 @@ instantiation, of which this module provides three different variants: specifying its value. Note that, after the send_header calls are done, :meth:`end_headers` MUST BE called in order to complete the operation. + This method does not reject input containing CRLF sequences. + .. versionchanged:: 3.2 Headers are stored in an internal buffer. @@ -297,6 +299,8 @@ instantiation, of which this module provides three different variants: buffered and sent directly the output stream.If the *message* is not specified, the HTTP message corresponding the response *code* is sent. + This method does not reject *message* containing CRLF sequences. + .. versionadded:: 3.2 .. method:: end_headers() @@ -386,6 +390,14 @@ instantiation, of which this module provides three different variants: This will be ``"SimpleHTTP/" + __version__``, where ``__version__`` is defined at the module level. + .. attribute:: default_content_type + + Specifies the Content-Type header value sent when the MIME type + cannot be guessed from the file extension of the requested URL. + By default, it is set to ``'application/octet-stream'``. + + .. versionadded:: next + .. attribute:: extensions_map A dictionary mapping suffixes into MIME types, contains custom overrides @@ -463,9 +475,11 @@ such as using different index file names by overriding the class attribute Command-line interface ---------------------- -:mod:`http.server` can also be invoked directly using the :option:`-m` +:mod:`!http.server` can also be invoked directly using the :option:`-m` switch of the interpreter. The following example illustrates how to serve -files relative to the current directory:: +files relative to the current directory: + +.. code-block:: bash python -m http.server [OPTIONS] [port] @@ -476,7 +490,9 @@ The following options are accepted: .. option:: port The server listens to port 8000 by default. The default can be overridden - by passing the desired port number as an argument:: + by passing the desired port number as an argument: + + .. code-block:: bash python -m http.server 9000 @@ -485,7 +501,9 @@ The following options are accepted: Specifies a specific address to which it should bind. Both IPv4 and IPv6 addresses are supported. By default, the server binds itself to all interfaces. For example, the following command causes the server to bind - to localhost only:: + to localhost only: + + .. code-block:: bash python -m http.server --bind 127.0.0.1 @@ -498,7 +516,9 @@ The following options are accepted: Specifies a directory to which it should serve the files. By default, the server uses the current directory. For example, the following command - uses a specific directory:: + uses a specific directory: + + .. code-block:: bash python -m http.server --directory /tmp/ @@ -508,15 +528,31 @@ The following options are accepted: Specifies the HTTP version to which the server is conformant. By default, the server is conformant to HTTP/1.0. For example, the following command - runs an HTTP/1.1 conformant server:: + runs an HTTP/1.1 conformant server: + + .. code-block:: bash python -m http.server --protocol HTTP/1.1 .. versionadded:: 3.11 +.. option:: --content-type + + Specifies the default Content-Type HTTP header used when the MIME type + cannot be guessed from the URL's file extension. By default, the server + uses ``'application/octet-stream'``: + + .. code-block:: bash + + python -m http.server --content-type text/html + + .. versionadded:: next + .. option:: --tls-cert - Specifies a TLS certificate chain for HTTPS connections:: + Specifies a TLS certificate chain for HTTPS connections: + + .. code-block:: bash python -m http.server --tls-cert fullchain.pem @@ -532,14 +568,16 @@ The following options are accepted: .. option:: --tls-password-file - Specifies the password file for password-protected private keys:: + Specifies the password file for password-protected private keys: + + .. code-block:: bash python -m http.server \ --tls-cert cert.pem \ --tls-key key.pem \ --tls-password-file password.txt - This option requires `--tls-cert`` to be specified. + This option requires ``--tls-cert`` to be specified. .. versionadded:: 3.14 @@ -555,6 +593,11 @@ Security considerations requests, this makes it possible for files outside of the specified directory to be served. +Methods :meth:`BaseHTTPRequestHandler.send_header` and +:meth:`BaseHTTPRequestHandler.send_response_only` assume sanitized input +and do not perform input validation such as checking for the presence of CRLF +sequences. Untrusted input may result in HTTP Header injection attacks. + Earlier versions of Python did not scrub control characters from the log messages emitted to stderr from ``python -m http.server`` or the default :class:`BaseHTTPRequestHandler` ``.log_message`` diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index fabea611e0e..c7c30e5300c 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -3,8 +3,6 @@ IDLE --- Python editor and shell ================================ -.. moduleauthor:: Guido van Rossum - **Source code:** :source:`Lib/idlelib/` .. index:: @@ -13,7 +11,7 @@ IDLE --- Python editor and shell single: Integrated Development Environment .. - Remember to update Lib/idlelib/help.html with idlelib.help.copy_source() when modifying this file. + Remember to update Lib/idlelib/help.html with idlelib.help.copy_strip() when modifying this file. -------------- @@ -37,6 +35,10 @@ IDLE has the following features: * configuration, browsers, and other dialogs +The IDLE application is implemented in the :mod:`idlelib` package. + +.. include:: ../includes/optional-module.rst + Menus ----- @@ -88,7 +90,7 @@ Save Save As... Save the current window with a Save As dialog. The file saved becomes the - new associated file for the window. (If your file namager is set to hide + new associated file for the window. (If your file manager is set to hide extensions, the current extension will be omitted in the file name box. If the new filename has no '.', '.py' and '.txt' will be added for Python and text files, except that on macOS Aqua,'.py' is added for all files.) @@ -154,7 +156,7 @@ Go to Line Show Completions Open a scrollable list allowing selection of existing names. See - :ref:`Completions ` in the Editing and navigation section below. + :ref:`Completions ` in the Editing and Navigation section below. Expand Word Expand a prefix you have typed to match a full word in the same window; @@ -163,7 +165,7 @@ Expand Word Show Call Tip After an unclosed parenthesis for a function, open a small window with function parameter hints. See :ref:`Calltips ` in the - Editing and navigation section below. + Editing and Navigation section below. Show Surrounding Parens Highlight the surrounding parenthesis. @@ -174,9 +176,9 @@ Format menu (Editor window only) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Format Paragraph - Reformat the current blank-line-delimited paragraph in comment block or - multiline string or selected line in a string. All lines in the - paragraph will be formatted to less than N columns, where N defaults to 72. + Rewrap the text block containing the text insert cursor. + Avoid code lines. See :ref:`Format block` in the + Editing and Navigation section below. Indent Region Shift selected lines right by the indent width (default 4 spaces). @@ -204,9 +206,9 @@ New Indent Width Open a dialog to change indent width. The accepted default by the Python community is 4 spaces. -Strip Trailing Chitespace +Strip Trailing Whitespace Remove trailing space and other whitespace characters after the last - non-whitespace character of a line by applying str.rstrip to each line, + non-whitespace character of a line by applying :meth:`str.rstrip` to each line, including lines within multiline strings. Except for Shell windows, remove extra newlines at the end of the file. @@ -562,6 +564,20 @@ In an editor, import statements have no effect until one runs the file. One might want to run a file after writing import statements, after adding function definitions, or after opening an existing file. +.. _format-block: + +Format block +^^^^^^^^^^^^ + +Reformat Paragraph rewraps a block ('paragraph') of contiguous equally +indented non-blank comments, a similar block of text within a multiline +string, or a selected subset of either. +If needed, add a blank line to separate string from code. +Partial lines in a selection expand to complete lines. +The resulting lines have the same indent as before +but have maximum total length of N columns (characters). +Change the default N of 72 on the Window tab of IDLE Settings. + .. _code-context: Code Context @@ -657,7 +673,9 @@ looked for in the user's home directory. Statements in this file will be executed in the Tk namespace, so this file is not useful for importing functions to be used from IDLE's Python shell. -Command line usage +.. _idlelib-cli: + +Command-line usage ^^^^^^^^^^^^^^^^^^ .. program:: idle diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 2a12a0ca8e9..b29b02d3cf5 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -4,14 +4,6 @@ .. module:: imaplib :synopsis: IMAP4 protocol client (requires sockets). -.. moduleauthor:: Piers Lauder -.. sectionauthor:: Piers Lauder -.. revised by ESR, January 2000 -.. changes for IMAP4_SSL by Tino Lange , March 2002 -.. changes for IMAP4_stream by Piers Lauder , - November 2002 -.. changes for IMAP4 IDLE by Forest , August 2024 - **Source code:** :source:`Lib/imaplib.py` .. index:: @@ -29,7 +21,7 @@ note that the ``STATUS`` command is not supported in IMAP4. .. include:: ../includes/wasm-notavail.rst -Three classes are provided by the :mod:`imaplib` module, :class:`IMAP4` is the +Three classes are provided by the :mod:`!imaplib` module, :class:`IMAP4` is the base class: @@ -413,7 +405,7 @@ An :class:`IMAP4` instance has the following methods: the password. Will only work if the server ``CAPABILITY`` response includes the phrase ``AUTH=CRAM-MD5``. - .. versionchanged:: next + .. versionchanged:: 3.15 An :exc:`IMAP4.error` is raised if MD5 support is not available. diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 12014309e26..63de4f91f4b 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -18,11 +18,9 @@ the metadata of an installed `Distribution Package `_\s, modules, if any). Built in part on Python's import system, this library -intends to replace similar functionality in the `entry point -API`_ and `metadata API`_ of ``pkg_resources``. Along with -:mod:`importlib.resources`, -this package can eliminate the need to use the older and less efficient -``pkg_resources`` package. +provides the entry point and metadata APIs that were previously +exposed by the now-removed ``pkg_resources`` package. Along with +:mod:`importlib.resources`, it supersedes ``pkg_resources``. ``importlib.metadata`` operates on third-party *distribution packages* installed into Python's ``site-packages`` directory via tools such as @@ -125,8 +123,8 @@ Entry points :meth:`!select` method for comparison to the attributes of the individual entry point definitions. - Note: it is not currently possible to query for entry points based on - their :attr:`!EntryPoint.dist` attribute (as different :class:`!Distribution` + Note: to query for entry points based on :attr:`!EntryPoint.dist` attribute, + use :meth:`Distribution.entry_points` instead (as different :class:`Distribution` instances do not currently compare equal, even if they have the same attributes) .. class:: EntryPoints @@ -291,7 +289,7 @@ Distribution files .. function:: files(distribution_name) Return the full set of files contained within the named - distribution package. + distribution package as :class:`PackagePath` instances. Raises :exc:`PackageNotFoundError` if the named distribution package is not installed in the current Python environment. @@ -304,12 +302,22 @@ Distribution files A :class:`pathlib.PurePath` derived object with additional ``dist``, ``size``, and ``hash`` properties corresponding to the distribution - package's installation metadata for that file. + package's installation metadata for that file, also: + + .. method:: locate() + + If possible, return the concrete :class:`SimplePath` allowing to access data, + or raise a :exc:`NotImplementedError` otherwise. + +.. class:: SimplePath + + A protocol representing a minimal subset of :class:`pathlib.Path` that allows to + check if it ``exists()``, to traverse using ``joinpath()`` and ``parent``, + and to retrieve data using ``read_text()`` and ``read_bytes()``. The :func:`!files` function takes a `Distribution Package `_ -name and returns all of the files installed by this distribution. Each file is reported -as a :class:`PackagePath` instance. For example:: +name and returns all of the files installed by this distribution. For example:: >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] # doctest: +SKIP >>> util # doctest: +SKIP @@ -402,6 +410,18 @@ function is not reliable with such installs. Distributions ============= +While the module level API described above is the most common and convenient usage, +all that information is accessible from the :class:`Distribution` class. +:class:`!Distribution` is an abstract object that represents the metadata for +a Python `Distribution Package `_. +Get the concrete :class:`!Distribution` subclass instance for an installed +distribution package by calling the :func:`distribution` function:: + + >>> from importlib.metadata import distribution # doctest: +SKIP + >>> dist = distribution('wheel') # doctest: +SKIP + >>> type(dist) # doctest: +SKIP + + .. function:: distribution(distribution_name) Return a :class:`Distribution` instance describing the named @@ -410,6 +430,14 @@ Distributions Raises :exc:`PackageNotFoundError` if the named distribution package is not installed in the current Python environment. +Thus, an alternative way to get e.g. the version number is through the +:attr:`Distribution.version` attribute:: + + >>> dist.version # doctest: +SKIP + '0.32.3' + +The same applies for :func:`entry_points` and :func:`files`. + .. class:: Distribution Details of an installed distribution package. @@ -418,43 +446,85 @@ Distributions equal, even if they relate to the same installed distribution and accordingly have the same attributes. -While the module level API described above is the most common and convenient usage, -you can get all of that information from the :class:`!Distribution` class. -:class:`!Distribution` is an abstract object that represents the metadata for -a Python `Distribution Package `_. -You can get the concrete :class:`!Distribution` subclass instance for an installed -distribution package by calling the :func:`distribution` function:: + .. staticmethod:: at(path) + .. classmethod:: from_name(name) - >>> from importlib.metadata import distribution # doctest: +SKIP - >>> dist = distribution('wheel') # doctest: +SKIP - >>> type(dist) # doctest: +SKIP - + Return a :class:`!Distribution` instance at the given path or + with the given name. -Thus, an alternative way to get the version number is through the -:class:`!Distribution` instance:: + .. classmethod:: discover(*, context=None, **kwargs) - >>> dist.version # doctest: +SKIP - '0.32.3' + Returns an iterable of :class:`!Distribution` instances for all packages + (see distribution-discovery_). -There are all kinds of additional metadata available on :class:`!Distribution` -instances:: + The optional argument *context* is a :class:`DistributionFinder.Context` + instance, used to modify the search for distributions. Alternatively, + *kwargs* may contain keyword arguments for constructing a new + :class:`!DistributionFinder.Context`. - >>> dist.metadata['Requires-Python'] # doctest: +SKIP - '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' - >>> dist.metadata['License'] # doctest: +SKIP - 'MIT' + .. attribute:: metadata + :type: PackageMetadata -For editable packages, an ``origin`` property may present :pep:`610` -metadata:: + There are all kinds of additional metadata available on :class:`!Distribution` + instances as a :class:`PackageMetadata` instance:: - >>> dist.origin.url - 'file:///path/to/wheel-0.32.3.editable-py3-none-any.whl' + >>> dist.metadata['Requires-Python'] # doctest: +SKIP + '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' + >>> dist.metadata['License'] # doctest: +SKIP + 'MIT' -The full set of available metadata is not described here. -See the PyPA `Core metadata specification `_ for additional details. + The full set of available metadata is not described here. + See the PyPA `Core metadata specification `_ for additional details. -.. versionadded:: 3.13 - The ``.origin`` property was added. + .. attribute:: name + :type: str + .. attribute:: requires + :type: list[str] + .. attribute:: version + :type: str + + A few metadata fields are also available as shortcut properties. + + .. versionadded:: 3.10 + + The ``name`` shortcut was added. + + .. attribute:: origin + + For editable packages, an ``origin`` property may present :pep:`610` + metadata (for non-editable packages, ``origin`` is :const:`None`):: + + >>> dist.origin.url + 'file:///path/to/wheel-0.32.3.editable-py3-none-any.whl' + + The ``origin`` object follows the `Direct URL Data Structure + `_. + + .. versionadded:: 3.13 + + .. attribute:: entry_points + :type: EntryPoints + + The entry points provided by this distribution package. + + .. attribute:: files + :type: list[PackagePath] | None + + All files contained in this distribution package. + Like :func:`files`, this returns :const:`None` if there are no records. + + The following two abstract methods need to be implemented when implementing-custom-providers_: + + .. method:: locate_file(path) + + Like :meth:`!PackagePath.locate`, return a :class:`SimplePath` for the given path. + Takes a :class:`os.PathLike` or a :class:`str`. + + .. method:: read_text(filename) + + A shortcut for ``distribution.locate_file(filename).read_text()``. + +.. _distribution-discovery: Distribution Discovery ====================== @@ -466,6 +536,61 @@ This metadata finder search defaults to ``sys.path``, but varies slightly in how - ``importlib.metadata`` does not honor :class:`bytes` objects on ``sys.path``. - ``importlib.metadata`` will incidentally honor :py:class:`pathlib.Path` objects on ``sys.path`` even though such values will be ignored for imports. +.. class:: DistributionFinder + + A :class:`~importlib.abc.MetaPathFinder` subclass capable of discovering + installed distributions. + + Custom providers should implement this interface in order to + supply metadata. + + .. class:: Context(**kwargs) + + A :class:`!Context` gives a custom provider a means to + solicit additional details from the callers of distribution discovery + functions like :func:`distributions` or :meth:`Distribution.discover` + beyond :attr:`!.name` and :attr:`!.path` when searching + for distributions. + + For example, a provider could expose suites of packages in either a + "public" or "private" ``realm``. A caller of distribution discovery + functions may wish to query only for distributions in a particular realm + and could call ``distributions(realm="private")`` to signal to the + custom provider to only include distributions from that + realm. + + Each :class:`!DistributionFinder` must expect any parameters and should + attempt to honor the canonical parameters defined below when + appropriate. + + See the section on :ref:`implementing-custom-providers` for more details. + + .. attribute:: name + + Specific name for which a distribution finder should match. + + A :attr:`!.name` of ``None`` matches all distributions. + + .. attribute:: path + + A property providing the sequence of directory paths that a + distribution finder should search. + + Typically refers to Python installed package paths such as + "site-packages" directories and defaults to :attr:`sys.path`. + + +.. function:: distributions(**kwargs) + + Returns an iterable of :class:`Distribution` instances for all packages. + + The *kwargs* argument may contain either a keyword argument ``context``, a + :class:`DistributionFinder.Context` instance, or pass keyword arguments for + constructing a new :class:`!DistributionFinder.Context`. The + :class:`!DistributionFinder.Context` is used to modify the search for + distributions. + +.. _implementing-custom-providers: Implementing Custom Providers ============================= @@ -493,7 +618,7 @@ interface expected of finders by Python's import system. ``importlib.metadata`` extends this protocol by looking for an optional ``find_distributions`` callable on the finders from :data:`sys.meta_path` and presents this extended interface as the -``DistributionFinder`` abstract base class, which defines this abstract +:class:`DistributionFinder` abstract base class, which defines this abstract method:: @abc.abstractmethod @@ -502,14 +627,16 @@ method:: loading the metadata for packages for the indicated ``context``. """ -The ``DistributionFinder.Context`` object provides ``.path`` and ``.name`` -properties indicating the path to search and name to match and may -supply other relevant context sought by the consumer. +The :class:`DistributionFinder.Context` object provides +:attr:`~DistributionFinder.Context.path` and +:attr:`~DistributionFinder.Context.name` properties indicating the path to +search and name to match and may supply other relevant context sought by the +consumer. In practice, to support finding distribution package metadata in locations other than the file system, subclass -``Distribution`` and implement the abstract methods. Then from -a custom finder, return instances of this derived ``Distribution`` in the +:class:`!Distribution` and implement the abstract methods. Then from +a custom finder, return instances of this derived :class:`!Distribution` in the ``find_distributions()`` method. Example @@ -529,7 +656,7 @@ Imagine a custom finder that loads Python modules from a database:: That importer now presumably provides importable modules from a database, but it provides no metadata or entry points. For this custom importer to provide metadata, it would also need to implement -``DistributionFinder``:: +:class:`DistributionFinder`:: from importlib.metadata import DistributionFinder @@ -586,9 +713,5 @@ packages served by the ``DatabaseImporter``, assuming that the ``.entry_points`` attributes. The ``DatabaseDistribution`` may also provide other metadata files, like -``RECORD`` (required for ``Distribution.files``) or override the -implementation of ``Distribution.files``. See the source for more inspiration. - - -.. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points -.. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api +``RECORD`` (required for :attr:`!Distribution.files`) or override the +implementation of :attr:`!Distribution.files`. See the source for more inspiration. diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index 8253a33f591..45979c56912 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -63,11 +63,14 @@ If the resource does not concretely exist on the file system, raise :exc:`FileNotFoundError`. - .. method:: is_resource(name) + .. method:: is_resource(path) :abstractmethod: - Returns ``True`` if the named *name* is considered a resource. - :exc:`FileNotFoundError` is raised if *name* does not exist. + Returns ``True`` if the named *path* is considered a resource. + :exc:`FileNotFoundError` is raised if *path* does not exist. + + .. versionchanged:: 3.10 + The argument *name* was renamed to *path*. .. method:: contents() :abstractmethod: diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index 7a11f4fe069..653fa61420b 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -31,15 +31,13 @@ not** have to exist as physical files and directories on the file system: for example, a package and its resources can be imported from a zip file using :py:mod:`zipimport`. -.. note:: +.. warning:: - This module provides functionality similar to `pkg_resources - `_ `Basic - Resource Access - `_ - without the performance overhead of that package. This makes reading - resources included in packages easier, with more stable and consistent - semantics. + :mod:`importlib.resources` follows the same security model as the built-in + :func:`open` function. Passing untrusted inputs to the functions + in this module is unsafe. + +.. note:: The standalone backport of this module provides more information on `using importlib.resources @@ -73,12 +71,15 @@ for example, a package and its resources can be imported from a zip file using .. versionadded:: 3.9 .. versionchanged:: 3.12 - *package* parameter was renamed to *anchor*. *anchor* can now - be a non-package module and if omitted will default to the caller's - module. *package* is still accepted for compatibility but will raise - a :exc:`DeprecationWarning`. Consider passing the anchor positionally or - using ``importlib_resources >= 5.10`` for a compatible interface - on older Pythons. + *package* parameter was renamed to *anchor*. + *package* was still accepted, but deprecated. + + .. versionchanged:: 3.15 + *package* parameter was fully removed. *anchor* can now be a + non-package module and if omitted will default to the caller's module. + *package* is no longer accepted since Python 3.15. Consider passing the + anchor positionally or using ``importlib_resources >= 5.10`` for a + compatible interface on older Pythons. .. function:: as_file(traversable) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 7eb048fcfc2..0b76020eacc 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -4,9 +4,6 @@ .. module:: importlib :synopsis: The implementation of the import machinery. -.. moduleauthor:: Brett Cannon -.. sectionauthor:: Brett Cannon - .. versionadded:: 3.1 **Source code:** :source:`Lib/importlib/__init__.py` @@ -17,7 +14,7 @@ Introduction ------------ -The purpose of the :mod:`importlib` package is three-fold. +The purpose of the :mod:`!importlib` package is three-fold. One is to provide the implementation of the :keyword:`import` statement (and thus, by extension, the @@ -215,8 +212,8 @@ Functions in unexpected behavior. It's recommended to use the :class:`threading.Lock` or other synchronization primitives for thread-safe module reloading. -:mod:`importlib.abc` -- Abstract base classes related to import ---------------------------------------------------------------- +:mod:`!importlib.abc` -- Abstract base classes related to import +---------------------------------------------------------------- .. module:: importlib.abc :synopsis: Abstract base classes related to import @@ -226,7 +223,7 @@ Functions -------------- -The :mod:`importlib.abc` module contains all of the core abstract base classes +The :mod:`!importlib.abc` module contains all of the core abstract base classes used by :keyword:`import`. Some subclasses of the core abstract base classes are also provided to help in implementing the core ABCs. @@ -275,6 +272,28 @@ ABC hierarchy:: .. versionchanged:: 3.4 Returns ``None`` when called instead of :data:`NotImplemented`. + .. method:: discover(parent=None) + + An optional method which searches for possible specs with given *parent* + module spec. If *parent* is *None*, :meth:`MetaPathFinder.discover` will + search for top-level modules. + + Returns an iterable of possible specs. + + Raises :exc:`ValueError` if *parent* is not a package module. + + .. warning:: + This method can potentially yield a very large number of objects, and + it may carry out IO operations when computing these values. + + Because of this, it will generally be desirable to compute the result + values on-the-fly, as they are needed. As such, the returned object is + only guaranteed to be an :class:`iterable `, + instead of a :class:`list` or other + :class:`collection ` type. + + .. versionadded:: 3.15 + .. class:: PathEntryFinder @@ -307,6 +326,28 @@ ABC hierarchy:: :meth:`importlib.machinery.PathFinder.invalidate_caches` when invalidating the caches of all cached finders. + .. method:: discover(parent=None) + + An optional method which searches for possible specs with given *parent* + module spec. If *parent* is *None*, :meth:`PathEntryFinder.discover` will + search for top-level modules. + + Returns an iterable of possible specs. + + Raises :exc:`ValueError` if *parent* is not a package module. + + .. warning:: + This method can potentially yield a very large number of objects, and + it may carry out IO operations when computing these values. + + Because of this, it will generally be desirable to compute the result + values on-the-fly, as they are needed. As such, the returned object is + only guaranteed to be an :class:`iterable `, + instead of a :class:`list` or other + :class:`collection ` type. + + .. versionadded:: 3.15 + .. class:: Loader @@ -320,6 +361,9 @@ ABC hierarchy:: .. versionchanged:: 3.7 Introduced the optional :meth:`get_resource_reader` method. + .. versionchanged:: 3.15 + Removed the ``load_module()`` method. + .. method:: create_module(spec) A method that returns the module object to use when @@ -344,47 +388,6 @@ ABC hierarchy:: .. versionchanged:: 3.6 :meth:`create_module` must also be defined. - .. method:: load_module(fullname) - - A legacy method for loading a module. If the module cannot be - loaded, :exc:`ImportError` is raised, otherwise the loaded module is - returned. - - If the requested module already exists in :data:`sys.modules`, that - module should be used and reloaded. - Otherwise the loader should create a new module and insert it into - :data:`sys.modules` before any loading begins, to prevent recursion - from the import. If the loader inserted a module and the load fails, it - must be removed by the loader from :data:`sys.modules`; modules already - in :data:`sys.modules` before the loader began execution should be left - alone. - - The loader should set several attributes on the module - (note that some of these attributes can change when a module is - reloaded): - - - :attr:`module.__name__` - - :attr:`module.__file__` - - :attr:`module.__cached__` *(deprecated)* - - :attr:`module.__path__` - - :attr:`module.__package__` *(deprecated)* - - :attr:`module.__loader__` *(deprecated)* - - When :meth:`exec_module` is available then backwards-compatible - functionality is provided. - - .. versionchanged:: 3.4 - Raise :exc:`ImportError` when called instead of - :exc:`NotImplementedError`. Functionality provided when - :meth:`exec_module` is available. - - .. deprecated-removed:: 3.4 3.15 - The recommended API for loading a module is :meth:`exec_module` - (and :meth:`create_module`). Loaders should implement it instead of - :meth:`load_module`. The import machinery takes care of all the - other responsibilities of :meth:`load_module` when - :meth:`exec_module` is implemented. - .. class:: ResourceLoader @@ -459,7 +462,7 @@ ABC hierarchy:: .. versionchanged:: 3.4 Raises :exc:`ImportError` instead of :exc:`NotImplementedError`. - .. staticmethod:: source_to_code(data, path='') + .. staticmethod:: source_to_code(data, path='', fullname=None) Create a code object from Python source. @@ -471,24 +474,25 @@ ABC hierarchy:: With the subsequent code object one can execute it in a module by running ``exec(code, module.__dict__)``. + The optional argument *fullname* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax + warnings by module name. + .. versionadded:: 3.4 .. versionchanged:: 3.5 Made the method static. + .. versionadded:: 3.15 + Added the *fullname* parameter. + + .. method:: exec_module(module) Implementation of :meth:`Loader.exec_module`. .. versionadded:: 3.4 - .. method:: load_module(fullname) - - Implementation of :meth:`Loader.load_module`. - - .. deprecated-removed:: 3.4 3.15 - use :meth:`exec_module` instead. - .. class:: ExecutionLoader @@ -522,6 +526,9 @@ ABC hierarchy:: .. versionadded:: 3.3 + .. versionchanged:: 3.15 + Removed the ``load_module()`` method. + .. attribute:: name The name of the module the loader can handle. @@ -530,13 +537,6 @@ ABC hierarchy:: Path to the file of the module. - .. method:: load_module(fullname) - - Calls super's ``load_module()``. - - .. deprecated-removed:: 3.4 3.15 - Use :meth:`Loader.exec_module` instead. - .. method:: get_filename(fullname) :abstractmethod: @@ -568,6 +568,9 @@ ABC hierarchy:: optimization to speed up loading by removing the parsing step of Python's compiler, and so no bytecode-specific API is exposed. + .. versionchanged:: 3.15 + Removed the ``load_module()`` method. + .. method:: path_stats(path) Optional abstract method which returns a :class:`dict` containing @@ -621,13 +624,6 @@ ABC hierarchy:: .. versionadded:: 3.4 - .. method:: load_module(fullname) - - Concrete implementation of :meth:`Loader.load_module`. - - .. deprecated-removed:: 3.4 3.15 - Use :meth:`exec_module` instead. - .. method:: get_source(fullname) Concrete implementation of :meth:`InspectLoader.get_source`. @@ -641,174 +637,8 @@ ABC hierarchy:: itself does not end in ``__init__``. -.. class:: ResourceReader - - *Superseded by TraversableResources* - - An :term:`abstract base class` to provide the ability to read - *resources*. - - From the perspective of this ABC, a *resource* is a binary - artifact that is shipped within a package. Typically this is - something like a data file that lives next to the ``__init__.py`` - file of the package. The purpose of this class is to help abstract - out the accessing of such data files so that it does not matter if - the package and its data file(s) are stored e.g. in a zip file - versus on the file system. - - For any of methods of this class, a *resource* argument is - expected to be a :term:`path-like object` which represents - conceptually just a file name. This means that no subdirectory - paths should be included in the *resource* argument. This is - because the location of the package the reader is for, acts as the - "directory". Hence the metaphor for directories and file - names is packages and resources, respectively. This is also why - instances of this class are expected to directly correlate to - a specific package (instead of potentially representing multiple - packages or a module). - - Loaders that wish to support resource reading are expected to - provide a method called ``get_resource_reader(fullname)`` which - returns an object implementing this ABC's interface. If the module - specified by fullname is not a package, this method should return - :const:`None`. An object compatible with this ABC should only be - returned when the specified module is a package. - - .. versionadded:: 3.7 - - .. deprecated-removed:: 3.12 3.14 - Use :class:`importlib.resources.abc.TraversableResources` instead. - - .. method:: open_resource(resource) - :abstractmethod: - - Returns an opened, :term:`file-like object` for binary reading - of the *resource*. - - If the resource cannot be found, :exc:`FileNotFoundError` is - raised. - - .. method:: resource_path(resource) - :abstractmethod: - - Returns the file system path to the *resource*. - - If the resource does not concretely exist on the file system, - raise :exc:`FileNotFoundError`. - - .. method:: is_resource(name) - :abstractmethod: - - Returns ``True`` if the named *name* is considered a resource. - :exc:`FileNotFoundError` is raised if *name* does not exist. - - .. method:: contents() - :abstractmethod: - - Returns an :term:`iterable` of strings over the contents of - the package. Do note that it is not required that all names - returned by the iterator be actual resources, e.g. it is - acceptable to return names for which :meth:`is_resource` would - be false. - - Allowing non-resource names to be returned is to allow for - situations where how a package and its resources are stored - are known a priori and the non-resource names would be useful. - For instance, returning subdirectory names is allowed so that - when it is known that the package and resources are stored on - the file system then those subdirectory names can be used - directly. - - The abstract method returns an iterable of no items. - - -.. class:: Traversable - - An object with a subset of :class:`pathlib.Path` methods suitable for - traversing directories and opening files. - - For a representation of the object on the file-system, use - :meth:`importlib.resources.as_file`. - - .. versionadded:: 3.9 - - .. deprecated-removed:: 3.12 3.14 - Use :class:`importlib.resources.abc.Traversable` instead. - - .. attribute:: name - - Abstract. The base name of this object without any parent references. - - .. method:: iterdir() - :abstractmethod: - - Yield ``Traversable`` objects in ``self``. - - .. method:: is_dir() - :abstractmethod: - - Return ``True`` if ``self`` is a directory. - - .. method:: is_file() - :abstractmethod: - - Return ``True`` if ``self`` is a file. - - .. method:: joinpath(child) - :abstractmethod: - - Return Traversable child in ``self``. - - .. method:: __truediv__(child) - :abstractmethod: - - Return ``Traversable`` child in ``self``. - - .. method:: open(mode='r', *args, **kwargs) - :abstractmethod: - - *mode* may be 'r' or 'rb' to open as text or binary. Return a handle - suitable for reading (same as :attr:`pathlib.Path.open`). - - When opening as text, accepts encoding parameters such as those - accepted by :class:`io.TextIOWrapper`. - - .. method:: read_bytes() - - Read contents of ``self`` as bytes. - - .. method:: read_text(encoding=None) - - Read contents of ``self`` as text. - - -.. class:: TraversableResources - - An abstract base class for resource readers capable of serving - the :meth:`importlib.resources.files` interface. Subclasses - :class:`importlib.resources.abc.ResourceReader` and provides - concrete implementations of the :class:`importlib.resources.abc.ResourceReader`'s - abstract methods. Therefore, any loader supplying - :class:`importlib.abc.TraversableResources` also supplies ResourceReader. - - Loaders that wish to support resource reading are expected to - implement this interface. - - .. versionadded:: 3.9 - - .. deprecated-removed:: 3.12 3.14 - Use :class:`importlib.resources.abc.TraversableResources` instead. - - .. method:: files() - :abstractmethod: - - Returns a :class:`importlib.resources.abc.Traversable` object for the loaded - package. - - - -:mod:`importlib.machinery` -- Importers and path hooks ------------------------------------------------------- +:mod:`!importlib.machinery` -- Importers and path hooks +------------------------------------------------------- .. module:: importlib.machinery :synopsis: Importers and path hooks @@ -1013,6 +843,36 @@ find and load modules. :exc:`ImportError` is raised. +.. class:: NamespacePath(name, path, path_finder) + + Represents a :term:`namespace package`'s path (:attr:`module.__path__`). + + When its ``__path__`` value is accessed it will be recomputed if necessary. + This keeps it in-sync with the global state (:attr:`sys.modules`). + + The *name* argument is the name of the namespace module. + + The *path* argument is the initial path value. + + The *path_finder* argument is the callable used to recompute the path value. + The callable has the same signature as :meth:`importlib.abc.MetaPathFinder.find_spec`. + + When the parent's :attr:`module.__path__` attribute is updated, the path + value is recomputed. + + If the parent module is missing from :data:`sys.modules`, then + :exc:`ModuleNotFoundError` will be raised. + + For top-level modules, the parent module's path is :data:`sys.path`. + + .. note:: + + :meth:`PathFinder.invalidate_caches` invalidates :class:`NamespacePath`, + forcing the path value to be recomputed next time it is accessed. + + .. versionadded:: 3.15 + + .. class:: SourceFileLoader(fullname, path) A concrete implementation of :class:`importlib.abc.SourceLoader` by @@ -1021,6 +881,9 @@ find and load modules. .. versionadded:: 3.3 + .. versionchanged:: 3.15 + Removed the ``load_module()`` method. + .. attribute:: name The name of the module that this loader will handle. @@ -1041,15 +904,6 @@ find and load modules. Concrete implementation of :meth:`importlib.abc.SourceLoader.set_data`. - .. method:: load_module(name=None) - - Concrete implementation of :meth:`importlib.abc.Loader.load_module` where - specifying the name of the module to load is optional. - - .. deprecated-removed:: 3.6 3.15 - - Use :meth:`importlib.abc.Loader.exec_module` instead. - .. class:: SourcelessFileLoader(fullname, path) @@ -1063,6 +917,9 @@ find and load modules. .. versionadded:: 3.3 + .. versionchanged:: 3.15 + Removed the ``load_module()`` method. + .. attribute:: name The name of the module the loader will handle. @@ -1084,15 +941,6 @@ find and load modules. Returns ``None`` as bytecode files have no source when this loader is used. - .. method:: load_module(name=None) - - Concrete implementation of :meth:`importlib.abc.Loader.load_module` where - specifying the name of the module to load is optional. - - .. deprecated-removed:: 3.6 3.15 - - Use :meth:`importlib.abc.Loader.exec_module` instead. - .. class:: ExtensionFileLoader(fullname, path) @@ -1224,8 +1072,7 @@ find and load modules. .. attribute:: cached - The filename of a compiled version of the module's code - (see :attr:`module.__cached__`). + The filename of a compiled version of the module's code. The :term:`finder` should always set this attribute but it may be ``None`` for modules that do not need compiled code stored. @@ -1257,7 +1104,7 @@ find and load modules. To accommodate this requirement, when running on iOS, extension module binaries are *not* packaged as ``.so`` files on ``sys.path``, but as individual standalone frameworks. To discover those frameworks, this loader - is be registered against the ``.fwork`` file extension, with a ``.fwork`` + is registered against the ``.fwork`` file extension, with a ``.fwork`` file acting as a placeholder in the original location of the binary on ``sys.path``. The ``.fwork`` file contains the path of the actual binary in the ``Frameworks`` folder, relative to the app bundle. To allow for @@ -1306,8 +1153,8 @@ find and load modules. Path to the ``.fwork`` file for the extension module. -:mod:`importlib.util` -- Utility code for importers ---------------------------------------------------- +:mod:`!importlib.util` -- Utility code for importers +---------------------------------------------------- .. module:: importlib.util :synopsis: Utility code for importers @@ -1327,7 +1174,7 @@ an :term:`importer`. .. versionadded:: 3.4 -.. function:: cache_from_source(path, debug_override=None, *, optimization=None) +.. function:: cache_from_source(path, *, optimization=None) Return the :pep:`3147`/:pep:`488` path to the byte-compiled file associated with the source *path*. For example, if *path* is ``/foo/bar/baz.py`` the return @@ -1346,12 +1193,6 @@ an :term:`importer`. ``/foo/bar/__pycache__/baz.cpython-32.opt-2.pyc``. The string representation of *optimization* can only be alphanumeric, else :exc:`ValueError` is raised. - The *debug_override* parameter is deprecated and can be used to override - the system's value for ``__debug__``. A ``True`` value is the equivalent of - setting *optimization* to the empty string. A ``False`` value is the same as - setting *optimization* to ``1``. If both *debug_override* an *optimization* - are not ``None`` then :exc:`TypeError` is raised. - .. versionadded:: 3.4 .. versionchanged:: 3.5 @@ -1361,6 +1202,9 @@ an :term:`importer`. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.15 + The *debug_override* parameter was removed. + .. function:: source_from_cache(path) diff --git a/Doc/library/index.rst b/Doc/library/index.rst index 163e1679c65..8fc77be520d 100644 --- a/Doc/library/index.rst +++ b/Doc/library/index.rst @@ -43,6 +43,7 @@ the `Python Package Index `_. constants.rst stdtypes.rst exceptions.rst + threadsafety.rst text.rst binary.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 7f2930ccd55..e23449886a3 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -9,14 +9,11 @@ .. module:: inspect :synopsis: Extract information and source code from live objects. -.. moduleauthor:: Ka-Ping Yee -.. sectionauthor:: Ka-Ping Yee - **Source code:** :source:`Lib/inspect.py` -------------- -The :mod:`inspect` module provides several useful functions to help get +The :mod:`!inspect` module provides several useful functions to help get information about live objects such as modules, classes, methods, functions, tracebacks, frame objects, and code objects. For example, it can help you examine the contents of a class, retrieve the source code of a method, extract @@ -198,10 +195,6 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | | read more :ref:`here | | | | `| +-----------------+-------------------+---------------------------+ -| | co_lnotab | encoded mapping of line | -| | | numbers to bytecode | -| | | indices | -+-----------------+-------------------+---------------------------+ | | co_freevars | tuple of names of free | | | | variables (referenced via | | | | a function's closure) | @@ -253,12 +246,21 @@ attributes (see :ref:`import-mod-attrs` for module attributes): +-----------------+-------------------+---------------------------+ | | gi_running | is the generator running? | +-----------------+-------------------+---------------------------+ +| | gi_suspended | is the generator | +| | | suspended? | ++-----------------+-------------------+---------------------------+ | | gi_code | code | +-----------------+-------------------+---------------------------+ | | gi_yieldfrom | object being iterated by | | | | ``yield from``, or | | | | ``None`` | +-----------------+-------------------+---------------------------+ +| | gi_state | state of the generator, | +| | | one of ``GEN_CREATED``, | +| | | ``GEN_RUNNING``, | +| | | ``GEN_SUSPENDED``, or | +| | | ``GEN_CLOSED`` | ++-----------------+-------------------+---------------------------+ | async generator | __name__ | name | +-----------------+-------------------+---------------------------+ | | __qualname__ | qualified name | @@ -270,8 +272,18 @@ attributes (see :ref:`import-mod-attrs` for module attributes): +-----------------+-------------------+---------------------------+ | | ag_running | is the generator running? | +-----------------+-------------------+---------------------------+ +| | ag_suspended | is the generator | +| | | suspended? | ++-----------------+-------------------+---------------------------+ | | ag_code | code | +-----------------+-------------------+---------------------------+ +| | ag_state | state of the async | +| | | generator, one of | +| | | ``AGEN_CREATED``, | +| | | ``AGEN_RUNNING``, | +| | | ``AGEN_SUSPENDED``, or | +| | | ``AGEN_CLOSED`` | ++-----------------+-------------------+---------------------------+ | coroutine | __name__ | name | +-----------------+-------------------+---------------------------+ | | __qualname__ | qualified name | @@ -283,12 +295,21 @@ attributes (see :ref:`import-mod-attrs` for module attributes): +-----------------+-------------------+---------------------------+ | | cr_running | is the coroutine running? | +-----------------+-------------------+---------------------------+ +| | cr_suspended | is the coroutine | +| | | suspended? | ++-----------------+-------------------+---------------------------+ | | cr_code | code | +-----------------+-------------------+---------------------------+ | | cr_origin | where coroutine was | | | | created, or ``None``. See | | | | |coroutine-origin-link| | +-----------------+-------------------+---------------------------+ +| | cr_state | state of the coroutine, | +| | | one of ``CORO_CREATED``, | +| | | ``CORO_RUNNING``, | +| | | ``CORO_SUSPENDED``, or | +| | | ``CORO_CLOSED`` | ++-----------------+-------------------+---------------------------+ | builtin | __doc__ | documentation string | +-----------------+-------------------+---------------------------+ | | __name__ | original name of this | @@ -316,10 +337,27 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Add ``__builtins__`` attribute to functions. +.. versionchanged:: 3.11 + + Add ``gi_suspended`` attribute to generators. + +.. versionchanged:: 3.11 + + Add ``cr_suspended`` attribute to coroutines. + +.. versionchanged:: 3.12 + + Add ``ag_suspended`` attribute to async generators. + .. versionchanged:: 3.14 Add ``f_generator`` attribute to frames. +.. versionchanged:: 3.15 + + Add ``gi_state`` attribute to generators, ``cr_state`` attribute to + coroutines, and ``ag_state`` attribute to async generators. + .. function:: getmembers(object[, predicate]) Return all the members of an object in a list of ``(name, value)`` @@ -503,7 +541,7 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. versionchanged:: 3.13 Functions wrapped in :func:`functools.partialmethod` now return ``True`` - if the wrapped function is a :term:`coroutine function`. + if the wrapped function is a :term:`asynchronous generator` function. .. function:: isasyncgen(object) @@ -616,17 +654,29 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Retrieving source code ---------------------- -.. function:: getdoc(object) +.. function:: getdoc(object, *, inherit_class_doc=True, fallback_to_class_doc=True) Get the documentation string for an object, cleaned up with :func:`cleandoc`. - If the documentation string for an object is not provided and the object is - a class, a method, a property or a descriptor, retrieve the documentation - string from the inheritance hierarchy. + If the documentation string for an object is not provided: + + * if the object is a class and *inherit_class_doc* is true (by default), + retrieve the documentation string from the inheritance hierarchy; + * if the object is a method, a property or a descriptor, retrieve + the documentation string from the inheritance hierarchy; + * otherwise, if *fallback_to_class_doc* is true (by default), retrieve + the documentation string from the class of the object. + Return ``None`` if the documentation string is invalid or missing. .. versionchanged:: 3.5 Documentation strings are now inherited if not overridden. + .. versionchanged:: 3.15 + Added parameters *inherit_class_doc* and *fallback_to_class_doc*. + + Documentation strings on :class:`~functools.cached_property` + objects are now inherited if not overridden. + .. function:: getcomments(object) @@ -1734,7 +1784,7 @@ which is a bitmap of the following flags: The flags are specific to CPython, and may not be defined in other Python implementations. Furthermore, the flags are an implementation detail, and can be removed or deprecated in future Python releases. - It's recommended to use public APIs from the :mod:`inspect` module + It's recommended to use public APIs from the :mod:`!inspect` module for any introspection needs. @@ -1773,10 +1823,10 @@ Buffer flags .. _inspect-module-cli: -Command Line Interface +Command-line interface ---------------------- -The :mod:`inspect` module also provides a basic introspection capability +The :mod:`!inspect` module also provides a basic introspection capability from the command line. .. program:: inspect diff --git a/Doc/library/io.rst b/Doc/library/io.rst index dfebccb5a9c..494e57fe1c0 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -4,14 +4,6 @@ .. module:: io :synopsis: Core tools for working with streams. -.. moduleauthor:: Guido van Rossum -.. moduleauthor:: Mike Verdone -.. moduleauthor:: Mark Russell -.. moduleauthor:: Antoine Pitrou -.. moduleauthor:: Amaury Forgeot d'Arc -.. moduleauthor:: Benjamin Peterson -.. sectionauthor:: Benjamin Peterson - **Source code:** :source:`Lib/io.py` -------------- @@ -24,7 +16,7 @@ Overview .. index:: single: file object; io module -The :mod:`io` module provides Python's main facilities for dealing with various +The :mod:`!io` module provides Python's main facilities for dealing with various types of I/O. There are three main types of I/O: *text I/O*, *binary I/O* and *raw I/O*. These are generic categories, and various backing stores can be used for each of them. A concrete object belonging to any of these @@ -292,7 +284,7 @@ interface to a buffered raw stream (:class:`BufferedIOBase`). Finally, Argument names are not part of the specification, and only the arguments of :func:`open` are intended to be used as keyword arguments. -The following table summarizes the ABCs provided by the :mod:`io` module: +The following table summarizes the ABCs provided by the :mod:`!io` module: .. tabularcolumns:: |l|l|L|L| @@ -587,7 +579,7 @@ I/O Base Classes When the underlying raw stream is non-blocking, implementations may either raise :exc:`BlockingIOError` or return ``None`` if no data is - available. :mod:`io` implementations return ``None``. + available. :mod:`!io` implementations return ``None``. .. method:: read1(size=-1, /) @@ -600,7 +592,7 @@ I/O Base Classes When the underlying raw stream is non-blocking, implementations may either raise :exc:`BlockingIOError` or return ``None`` if no data is - available. :mod:`io` implementations return ``None``. + available. :mod:`!io` implementations return ``None``. .. method:: readinto(b, /) @@ -720,7 +712,7 @@ than raw I/O does. contains initial data. Methods may be used from multiple threads without external locking in - :term:`free threading` builds. + :term:`free-threaded builds `. :class:`BytesIO` provides or overrides these methods in addition to those from :class:`BufferedIOBase` and :class:`IOBase`: diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 9e887d8e657..ba20310d63b 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -4,13 +4,11 @@ .. module:: ipaddress :synopsis: IPv4/IPv6 manipulation library. -.. moduleauthor:: Peter Moody - **Source code:** :source:`Lib/ipaddress.py` -------------- -:mod:`ipaddress` provides the capabilities to create, manipulate and +:mod:`!ipaddress` provides the capabilities to create, manipulate and operate on IPv4 and IPv6 addresses and networks. The functions and classes in this module make it straightforward to handle @@ -34,7 +32,7 @@ This is the full module API reference—for an overview and introduction, see Convenience factory functions ----------------------------- -The :mod:`ipaddress` module provides factory functions to conveniently create +The :mod:`!ipaddress` module provides factory functions to conveniently create IP addresses, networks and interfaces: .. function:: ip_address(address) @@ -1027,7 +1025,7 @@ The module also provides the following module level functions: IPv4Address('192.0.2.0') <= IPv4Network('192.0.2.0/24') doesn't make sense. There are some times however, where you may wish to - have :mod:`ipaddress` sort these anyway. If you need to do this, you can use + have :mod:`!ipaddress` sort these anyway. If you need to do this, you can use this function as the *key* argument to :func:`sorted`. *obj* is either a network or address object. diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index aa46920d352..06f8bf2a8b6 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -4,9 +4,6 @@ .. module:: itertools :synopsis: Functions creating iterators for efficient looping. -.. moduleauthor:: Raymond Hettinger -.. sectionauthor:: Raymond Hettinger - .. testsetup:: from itertools import * @@ -30,32 +27,24 @@ For instance, SML provides a tabulation tool: ``tabulate(f)`` which produces a sequence ``f(0), f(1), ...``. The same effect can be achieved in Python by combining :func:`map` and :func:`count` to form ``map(f, count())``. - -**Infinite iterators:** - -================== ================= ================================================= ========================================= -Iterator Arguments Results Example -================== ================= ================================================= ========================================= -:func:`count` [start[, step]] start, start+step, start+2*step, ... ``count(10) → 10 11 12 13 14 ...`` -:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') → A B C D A B C D ...`` -:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) → 10 10 10`` -================== ================= ================================================= ========================================= - -**Iterators terminating on the shortest input sequence:** +**General iterators:** ============================ ============================ ================================================= ============================================================= Iterator Arguments Results Example ============================ ============================ ================================================= ============================================================= :func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) → 1 3 6 10 15`` -:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=2) → AB CD EF G`` +:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=3) → ABC DEF G`` :func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') → A B C D E F`` :func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) → A B C D E F`` :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) → A C E F`` +:func:`count` [start[, step]] start, start+step, start+2*step, ... ``count(10) → 10 11 12 13 14 ...`` +:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') → A B C D A B C D ...`` :func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8`` :func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8`` :func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) ``groupby(['A','B','DEF'], len) → (1, A B) (3, DEF)`` :func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) → C D E F G`` :func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') → AB BC CD DE EF FG`` +:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) → 10 10 10`` :func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) → 32 9 1000`` :func:`takewhile` predicate, seq seq[0], seq[1], until predicate fails ``takewhile(lambda x: x<5, [1,4,6,3,8]) → 1 4`` :func:`tee` it, n it1, it2, ... itn splits one iterator into n ``tee('ABC', 2) → A B C, A B C`` @@ -181,7 +170,7 @@ loops that truncate the stream. Roughly equivalent to:: def batched(iterable, n, *, strict=False): - # batched('ABCDEFG', 2) → AB CD EF G + # batched('ABCDEFG', 3) → ABC DEF G if n < 1: raise ValueError('n must be at least one') iterator = iter(iterable) @@ -819,7 +808,7 @@ well as with the built-in itertools such as ``map()``, ``filter()``, A secondary purpose of the recipes is to serve as an incubator. The ``accumulate()``, ``compress()``, and ``pairwise()`` itertools started out as -recipes. Currently, the ``sliding_window()``, ``iter_index()``, and ``sieve()`` +recipes. Currently, the ``sliding_window()``, ``derangements()``, and ``sieve()`` recipes are being tested to see whether they prove their worth. Substantially all of these recipes and many, many others can be installed from @@ -838,11 +827,18 @@ and :term:`generators ` which incur interpreter overhead. .. testcode:: + from itertools import (accumulate, batched, chain, combinations, compress, + count, cycle, filterfalse, groupby, islice, permutations, product, + repeat, starmap, tee, zip_longest) from collections import Counter, deque from contextlib import suppress from functools import reduce - from math import comb, prod, sumprod, isqrt - from operator import itemgetter, getitem, mul, neg + from heapq import heappush, heappushpop, heappush_max, heappushpop_max + from math import comb, isqrt, prod, sumprod + from operator import getitem, is_not, itemgetter, mul, neg, truediv + + + # ==== Basic one liners ==== def take(n, iterable): "Return first n items of the iterable as a list." @@ -853,10 +849,6 @@ and :term:`generators ` which incur interpreter overhead. # prepend(1, [2, 3, 4]) → 1 2 3 4 return chain([value], iterable) - def tabulate(function, start=0): - "Return function(0), function(1), ..." - return map(function, count(start)) - def repeatfunc(function, times=None, *args): "Repeat calls to a function with specified arguments." if times is None: @@ -899,8 +891,8 @@ and :term:`generators ` which incur interpreter overhead. def first_true(iterable, default=False, predicate=None): "Returns the first true value or the *default* if there is no true value." - # first_true([a,b,c], x) → a or b or c or x - # first_true([a,b], x, f) → a if f(a) else b if f(b) else x + # first_true([a, b, c], x) → a or b or c or x + # first_true([a, b], x, f) → a if f(a) else b if f(b) else x return next(filter(predicate, iterable), default) def all_equal(iterable, key=None): @@ -908,6 +900,9 @@ and :term:`generators ` which incur interpreter overhead. # all_equal('4٤௪౪໔', key=int) → True return len(take(2, groupby(iterable, key))) <= 1 + + # ==== Data pipelines ==== + def unique_justseen(iterable, key=None): "Yield unique elements, preserving order. Remember only the element just seen." # unique_justseen('AAAABBBCCDAABBB') → A B C D A B @@ -933,14 +928,14 @@ and :term:`generators ` which incur interpreter overhead. yield element def unique(iterable, key=None, reverse=False): - "Yield unique elements in sorted order. Supports unhashable inputs." - # unique([[1, 2], [3, 4], [1, 2]]) → [1, 2] [3, 4] - sequenced = sorted(iterable, key=key, reverse=reverse) - return unique_justseen(sequenced, key=key) + "Yield unique elements in sorted order. Supports unhashable inputs." + # unique([[1, 2], [3, 4], [1, 2]]) → [1, 2] [3, 4] + sequenced = sorted(iterable, key=key, reverse=reverse) + return unique_justseen(sequenced, key=key) def sliding_window(iterable, n): "Collect data into overlapping fixed-length chunks or blocks." - # sliding_window('ABCDEFG', 4) → ABCD BCDE CDEF DEFG + # sliding_window('ABCDEFG', 3) → ABC BCD CDE DEF EFG iterator = iter(iterable) window = deque(islice(iterator, n - 1), maxlen=n) for x in iterator: @@ -949,7 +944,7 @@ and :term:`generators ` which incur interpreter overhead. def grouper(iterable, n, *, incomplete='fill', fillvalue=None): "Collect data into non-overlapping fixed-length chunks or blocks." - # grouper('ABCDEFG', 3, fillvalue='x') → ABC DEF Gxx + # grouper('ABCDEFG', 3, fillvalue='x') → ABC DEF Gxx # grouper('ABCDEFG', 3, incomplete='strict') → ABC DEF ValueError # grouper('ABCDEFG', 3, incomplete='ignore') → ABC DEF iterators = [iter(iterable)] * n @@ -978,6 +973,16 @@ and :term:`generators ` which incur interpreter overhead. slices = starmap(slice, combinations(range(len(seq) + 1), 2)) return map(getitem, repeat(seq), slices) + def derangements(iterable, r=None): + "Produce r length permutations without fixed points." + # derangements('ABCD') → BADC BCDA BDAC CADB CDAB CDBA DABC DCAB DCBA + # Algorithm credited to Stefan Pochmann + seq = tuple(iterable) + pos = tuple(range(len(seq))) + have_moved = map(map, repeat(is_not), repeat(pos), permutations(pos, r=r)) + valid_derangements = map(all, have_moved) + return compress(permutations(seq, r=r), valid_derangements) + def iter_index(iterable, value, start=0, stop=None): "Return indices where a value occurs in a sequence or iterable." # iter_index('AABCADEAF', 'A') → 0 1 4 7 @@ -1005,9 +1010,7 @@ and :term:`generators ` which incur interpreter overhead. yield function() -The following recipes have a more mathematical flavor: - -.. testcode:: + # ==== Mathematical operations ==== def multinomial(*counts): "Number of distinct arrangements of a multiset." @@ -1026,9 +1029,12 @@ The following recipes have a more mathematical flavor: # sum_of_squares([10, 20, 30]) → 1400 return sumprod(*tee(iterable)) + + # ==== Matrix operations ==== + def reshape(matrix, columns): "Reshape a 2-D matrix to have a given number of columns." - # reshape([(0, 1), (2, 3), (4, 5)], 3) → (0, 1, 2), (3, 4, 5) + # reshape([(0, 1), (2, 3), (4, 5)], 3) → (0, 1, 2) (3, 4, 5) return batched(chain.from_iterable(matrix), columns, strict=True) def transpose(matrix): @@ -1038,10 +1044,13 @@ The following recipes have a more mathematical flavor: def matmul(m1, m2): "Multiply two matrices." - # matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)]) → (49, 80), (41, 60) + # matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)]) → (49, 80) (41, 60) n = len(m2[0]) return batched(starmap(sumprod, product(m1, transpose(m2))), n) + + # ==== Polynomial arithmetic ==== + def convolve(signal, kernel): """Discrete linear convolution of two iterables. Equivalent to polynomial multiplication. @@ -1096,6 +1105,9 @@ The following recipes have a more mathematical flavor: powers = reversed(range(1, n)) return list(map(mul, coefficients, powers)) + + # ==== Number theory ==== + def sieve(n): "Primes less than n." # sieve(30) → 2 3 5 7 11 13 17 19 23 29 @@ -1134,6 +1146,49 @@ The following recipes have a more mathematical flavor: return n + # ==== Running statistics ==== + + def running_mean(iterable): + "Average of values seen so far." + # running_mean([37, 33, 38, 28]) → 37 35 36 34 + return map(truediv, accumulate(iterable), count(1)) + + def running_min(iterable): + "Smallest of values seen so far." + # running_min([37, 33, 38, 28]) → 37 33 33 28 + return accumulate(iterable, func=min) + + def running_max(iterable): + "Largest of values seen so far." + # running_max([37, 33, 38, 28]) → 37 37 38 38 + return accumulate(iterable, func=max) + + def running_median(iterable): + "Median of values seen so far." + # running_median([37, 33, 38, 28]) → 37 35 37 35 + read = iter(iterable).__next__ + lo = [] # max-heap + hi = [] # min-heap the same size as or one smaller than lo + with suppress(StopIteration): + while True: + heappush_max(lo, heappushpop(hi, read())) + yield lo[0] + heappush(hi, heappushpop_max(lo, read())) + yield (lo[0] + hi[0]) / 2 + + def running_statistics(iterable): + "Aggregate statistics for values seen so far." + # Generate tuples: (size, minimum, median, maximum, mean) + t0, t1, t2, t3 = tee(iterable, 4) + return zip( + count(1), + running_min(t0), + running_median(t1), + running_max(t2), + running_mean(t3), + ) + + .. doctest:: :hide: @@ -1210,10 +1265,6 @@ The following recipes have a more mathematical flavor: [(0, 'a'), (1, 'b'), (2, 'c')] - >>> list(islice(tabulate(lambda x: 2*x), 4)) - [0, 2, 4, 6] - - >>> for _ in loops(5): ... print('hi') ... @@ -1663,6 +1714,36 @@ The following recipes have a more mathematical flavor: ['A', 'AB', 'ABC', 'ABCD', 'B', 'BC', 'BCD', 'C', 'CD', 'D'] + >>> ' '.join(map(''.join, derangements('ABCD'))) + 'BADC BCDA BDAC CADB CDAB CDBA DABC DCAB DCBA' + >>> ' '.join(map(''.join, derangements('ABCD', 3))) + 'BAD BCA BCD BDA CAB CAD CDA CDB DAB DCA DCB' + >>> ' '.join(map(''.join, derangements('ABCD', 2))) + 'BA BC BD CA CD DA DC' + >>> ' '.join(map(''.join, derangements('ABCD', 1))) + 'B C D' + >>> ' '.join(map(''.join, derangements('ABCD', 0))) + '' + >>> # Compare number of derangements to https://oeis.org/A000166 + >>> [len(list(derangements(range(n)))) for n in range(10)] + [1, 0, 1, 2, 9, 44, 265, 1854, 14833, 133496] + >>> # Verify that identical objects are treated as unique by position + >>> identical = 'X' + >>> distinct = 'x' + >>> seq1 = ('A', identical, 'B', identical) + >>> result1 = ' '.join(map(''.join, derangements(seq1))) + >>> result1 + 'XAXB XBXA XXAB BAXX BXAX BXXA XAXB XBAX XBXA' + >>> seq2 = ('A', identical, 'B', distinct) + >>> result2 = ' '.join(map(''.join, derangements(seq2))) + >>> result2 + 'XAxB XBxA XxAB BAxX BxAX BxXA xAXB xBAX xBXA' + >>> result1 == result2 + False + >>> result1.casefold() == result2.casefold() + True + + >>> list(powerset([1,2,3])) [(), (1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)] >>> all(len(list(powerset(range(n)))) == 2**n for n in range(18)) @@ -1743,11 +1824,37 @@ The following recipes have a more mathematical flavor: True + >>> list(running_mean([8.5, 9.5, 7.5, 6.5])) + [8.5, 9.0, 8.5, 8.0] + >>> list(running_mean([37, 33, 38, 28])) + [37.0, 35.0, 36.0, 34.0] + + + >>> list(running_min([37, 33, 38, 28])) + [37, 33, 33, 28] + + + >>> list(running_max([37, 33, 38, 28])) + [37, 37, 38, 38] + + + >>> list(running_median([37, 33, 38, 28])) + [37, 35.0, 37, 35.0] + + + >>> list(running_statistics([37, 33, 38, 28])) + [(1, 37, 37, 37, 37.0), (2, 33, 35.0, 37, 35.0), (3, 33, 37, 38, 36.0), (4, 28, 35.0, 38, 34.0)] + + .. testcode:: :hide: # Old recipes and their tests which are guaranteed to continue to work. + def tabulate(function, start=0): + "Return function(0), function(1), ..." + return map(function, count(start)) + def old_sumprod_recipe(vec1, vec2): "Compute a sum of products." return sum(starmap(operator.mul, zip(vec1, vec2, strict=True))) @@ -1827,6 +1934,10 @@ The following recipes have a more mathematical flavor: .. doctest:: :hide: + >>> list(islice(tabulate(lambda x: 2*x), 4)) + [0, 2, 4, 6] + + >>> dotproduct([1,2,3], [4,5,6]) 32 diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 12a5a96a3c5..b354e7ba534 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -4,9 +4,6 @@ .. module:: json :synopsis: Encode and decode the JSON format. -.. moduleauthor:: Bob Ippolito -.. sectionauthor:: Bob Ippolito - **Source code:** :source:`Lib/json/__init__.py` -------------- @@ -121,7 +118,7 @@ Extending :class:`JSONEncoder`:: ['[2.0', ', 1.0', ']'] -Using :mod:`json` from the shell to validate and pretty-print: +Using :mod:`!json` from the shell to validate and pretty-print: .. code-block:: shell-session @@ -183,8 +180,10 @@ Basic Usage :param bool ensure_ascii: If ``True`` (the default), the output is guaranteed to - have all incoming non-ASCII characters escaped. - If ``False``, these characters will be outputted as-is. + have all incoming non-ASCII and non-printable characters escaped. + If ``False``, all characters will be outputted as-is, except for + the characters that must be escaped: quotation mark, reverse solidus, + and the control characters U+0000 through U+001F. :param bool check_circular: If ``False``, the circular reference check for container types is skipped @@ -265,7 +264,7 @@ Basic Usage .. function:: load(fp, *, cls=None, object_hook=None, parse_float=None, \ parse_int=None, parse_constant=None, \ - object_pairs_hook=None, **kw) + object_pairs_hook=None, array_hook=None, **kw) Deserialize *fp* to a Python object using the :ref:`JSON-to-Python conversion table `. @@ -302,6 +301,15 @@ Basic Usage Default ``None``. :type object_pairs_hook: :term:`callable` | None + :param array_hook: + If set, a function that is called with the result of + any JSON array literal decoded with as a Python list. + The return value of this function will be used + instead of the :class:`list`. + This feature can be used to implement custom decoders. + Default ``None``. + :type array_hook: :term:`callable` | None + :param parse_float: If set, a function that is called with the string of every JSON float to be decoded. @@ -350,7 +358,10 @@ Basic Usage conversion length limitation ` to help avoid denial of service attacks. -.. function:: loads(s, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) + .. versionchanged:: 3.15 + Added the optional *array_hook* parameter. + +.. function:: loads(s, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, array_hook=None, **kw) Identical to :func:`load`, but instead of a file-like object, deserialize *s* (a :class:`str`, :class:`bytes` or :class:`bytearray` @@ -368,7 +379,7 @@ Basic Usage Encoders and Decoders --------------------- -.. class:: JSONDecoder(*, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, strict=True, object_pairs_hook=None) +.. class:: JSONDecoder(*, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, strict=True, object_pairs_hook=None, array_hook=None) Simple JSON decoder. @@ -413,6 +424,14 @@ Encoders and Decoders .. versionchanged:: 3.1 Added support for *object_pairs_hook*. + *array_hook* is an optional function that will be called with the + result of every JSON array decoded as a list. The return value of + *array_hook* will be used instead of the :class:`list`. This feature can be + used to implement custom decoders. + + .. versionchanged:: 3.15 + Added support for *array_hook*. + *parse_float* is an optional function that will be called with the string of every JSON float to be decoded. By default, this is equivalent to ``float(num_str)``. This can be used to use another datatype or parser for @@ -495,8 +514,10 @@ Encoders and Decoders :class:`bool` or ``None``. If *skipkeys* is true, such items are simply skipped. If *ensure_ascii* is true (the default), the output is guaranteed to - have all incoming non-ASCII characters escaped. If *ensure_ascii* is - false, these characters will be output as-is. + have all incoming non-ASCII and non-printable characters escaped. + If *ensure_ascii* is false, all characters will be output as-is, except for + the characters that must be escaped: quotation mark, reverse solidus, + and the control characters U+0000 through U+001F. If *check_circular* is true (the default), then lists, dicts, and custom encoded objects will be checked for circular references during encoding to @@ -636,7 +657,7 @@ UTF-32, with UTF-8 being the recommended default for maximum interoperability. As permitted, though not required, by the RFC, this module's serializer sets *ensure_ascii=True* by default, thus escaping the output so that the resulting -strings only contain ASCII characters. +strings only contain printable ASCII characters. Other than the *ensure_ascii* parameter, this module is defined strictly in terms of conversion between Python objects and @@ -743,8 +764,8 @@ Command-line interface -------------- -The :mod:`json` module can be invoked as a script via ``python -m json`` -to validate and pretty-print JSON objects. The :mod:`json.tool` submodule +The :mod:`!json` module can be invoked as a script via ``python -m json`` +to validate and pretty-print JSON objects. The :mod:`!json.tool` submodule implements this interface. If the optional ``infile`` and ``outfile`` arguments are not @@ -765,7 +786,7 @@ specified, :data:`sys.stdin` and :data:`sys.stdout` will be used respectively: alphabetically by key. .. versionchanged:: 3.14 - The :mod:`json` module may now be directly executed as + The :mod:`!json` module may now be directly executed as ``python -m json``. For backwards compatibility, invoking the CLI as ``python -m json.tool`` remains supported. diff --git a/Doc/library/linecache.rst b/Doc/library/linecache.rst index e766a928094..ff07499e58f 100644 --- a/Doc/library/linecache.rst +++ b/Doc/library/linecache.rst @@ -4,13 +4,11 @@ .. module:: linecache :synopsis: Provides random access to individual lines from text files. -.. sectionauthor:: Moshe Zadka - **Source code:** :source:`Lib/linecache.py` -------------- -The :mod:`linecache` module allows one to get any line from a Python source file, while +The :mod:`!linecache` module allows one to get any line from a Python source file, while attempting to optimize internally, using a cache, the common case where many lines are read from a single file. This is used by the :mod:`traceback` module to retrieve source lines for inclusion in the formatted traceback. @@ -19,7 +17,7 @@ The :func:`tokenize.open` function is used to open files. This function uses :func:`tokenize.detect_encoding` to get the encoding of the file; in the absence of an encoding token, the file encoding defaults to UTF-8. -The :mod:`linecache` module defines the following functions: +The :mod:`!linecache` module defines the following functions: .. function:: getline(filename, lineno, module_globals=None) @@ -31,7 +29,7 @@ The :mod:`linecache` module defines the following functions: .. index:: triple: module; search; path If *filename* indicates a frozen module (starting with ``' -.. sectionauthor:: Martin von Löwis - **Source code:** :source:`Lib/locale.py` -------------- -The :mod:`locale` module opens access to the POSIX locale database and +The :mod:`!locale` module opens access to the POSIX locale database and functionality. The POSIX locale mechanism allows programmers to deal with certain cultural issues in an application, without requiring the programmer to know all the specifics of each country where the software is executed. .. index:: pair: module; _locale -The :mod:`locale` module is implemented on top of the :mod:`!_locale` module, +The :mod:`!locale` module is implemented on top of the :mod:`!_locale` module, which in turn uses an ANSI C locale implementation if available. -The :mod:`locale` module defines the following exception and functions: +The :mod:`!locale` module defines the following exception and functions: .. exception:: Error @@ -48,8 +45,19 @@ The :mod:`locale` module defines the following exception and functions: If *locale* is omitted or ``None``, the current setting for *category* is returned. + Example:: + + >>> import locale + >>> loc = locale.setlocale(locale.LC_ALL) # get current locale + # use German locale; name and availability varies with platform + >>> locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8') + >>> locale.strcoll('f\xe4n', 'foo') # compare a string containing an umlaut + >>> locale.setlocale(locale.LC_ALL, '') # use user's preferred locale + >>> locale.setlocale(locale.LC_ALL, 'C') # use default (C) locale + >>> locale.setlocale(locale.LC_ALL, loc) # restore saved locale + :func:`setlocale` is not thread-safe on most systems. Applications typically - start with a call of :: + start with a call of:: import locale locale.setlocale(locale.LC_ALL, '') @@ -58,7 +66,7 @@ The :mod:`locale` module defines the following exception and functions: specified in the :envvar:`LANG` environment variable). If the locale is not changed thereafter, using multithreading should not cause problems. - .. versionchanged:: next + .. versionchanged:: 3.15 Support language codes with ``@``-modifiers. @@ -359,8 +367,6 @@ The :mod:`locale` module defines the following exception and functions: determined. The "C" locale is represented as ``(None, None)``. - .. deprecated-removed:: 3.11 3.15 - .. function:: getlocale(category=LC_CTYPE) @@ -374,7 +380,7 @@ The :mod:`locale` module defines the following exception and functions: determined. The "C" locale is represented as ``(None, None)``. - .. versionchanged:: next + .. versionchanged:: 3.15 ``@``-modifier are no longer silently removed, but included in the language code. @@ -524,14 +530,14 @@ The :mod:`locale` module defines the following exception and functions: SSH connections. Python doesn't internally use locale-dependent character transformation functions - from ``ctype.h``. Instead, an internal ``pyctype.h`` provides locale-independent - equivalents like :c:macro:`!Py_TOLOWER`. + from ``ctype.h``. Instead, ``pyctype.h`` provides locale-independent + equivalents like :c:macro:`Py_TOLOWER`. .. data:: LC_COLLATE Locale category for sorting strings. The functions :func:`strcoll` and - :func:`strxfrm` of the :mod:`locale` module are affected. + :func:`strxfrm` of the :mod:`!locale` module are affected. .. data:: LC_TIME @@ -560,7 +566,7 @@ The :mod:`locale` module defines the following exception and functions: .. data:: LC_NUMERIC Locale category for formatting numbers. The functions :func:`format_string`, - :func:`atoi`, :func:`atof` and :func:`.str` of the :mod:`locale` module are + :func:`atoi`, :func:`atof` and :func:`.str` of the :mod:`!locale` module are affected by that category. All other numeric formatting operations are not affected. @@ -580,18 +586,6 @@ The :mod:`locale` module defines the following exception and functions: :func:`localeconv`. -Example:: - - >>> import locale - >>> loc = locale.getlocale() # get current locale - # use German locale; name might vary with platform - >>> locale.setlocale(locale.LC_ALL, 'de_DE') - >>> locale.strcoll('f\xe4n', 'foo') # compare a string containing an umlaut - >>> locale.setlocale(locale.LC_ALL, '') # use user's preferred locale - >>> locale.setlocale(locale.LC_ALL, 'C') # use default (C) locale - >>> locale.setlocale(locale.LC_ALL, loc) # restore saved locale - - Background, details, hints, tips and caveats -------------------------------------------- @@ -696,7 +690,7 @@ the current locale is. But since the return value can only be used portably to restore it, that is not very useful (except perhaps to find out whether or not the locale is ``C``). -When Python code uses the :mod:`locale` module to change the locale, this also +When Python code uses the :mod:`!locale` module to change the locale, this also affects the embedding application. If the embedding application doesn't want this to happen, it should remove the :mod:`!_locale` extension module (which does all the work) from the table of built-in modules in the :file:`config.c` file, diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 96cca3073fe..30bf7860a75 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -4,9 +4,6 @@ .. module:: logging.config :synopsis: Configuration of the logging module. -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - **Source code:** :source:`Lib/logging/config.py` .. sidebar:: Important @@ -28,7 +25,7 @@ Configuration functions ^^^^^^^^^^^^^^^^^^^^^^^ The following functions configure the logging module. They are located in the -:mod:`logging.config` module. Their use is optional --- you can configure the +:mod:`!logging.config` module. Their use is optional --- you can configure the logging module using these functions or by making calls to the main API (defined in :mod:`logging` itself) and defining handlers which are declared either in :mod:`logging` or :mod:`logging.handlers`. @@ -55,7 +52,7 @@ in :mod:`logging` itself) and defining handlers which are declared either in Parsing is performed by the :class:`DictConfigurator` class, whose constructor is passed the dictionary used for configuration, and - has a :meth:`configure` method. The :mod:`logging.config` module + has a :meth:`configure` method. The :mod:`!logging.config` module has a callable attribute :attr:`dictConfigClass` which is initially set to :class:`DictConfigurator`. You can replace the value of :attr:`dictConfigClass` with a diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index d74ef73ee28..714db5fa12a 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -4,9 +4,6 @@ .. module:: logging.handlers :synopsis: Handlers for the logging module. -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - **Source code:** :source:`Lib/logging/handlers.py` .. sidebar:: Important @@ -160,7 +157,7 @@ WatchedFileHandler .. currentmodule:: logging.handlers -The :class:`WatchedFileHandler` class, located in the :mod:`logging.handlers` +The :class:`WatchedFileHandler` class, located in the :mod:`!logging.handlers` module, is a :class:`FileHandler` which watches the file it is logging to. If the file changes, it is closed and reopened using the file name. @@ -213,7 +210,7 @@ for this value. BaseRotatingHandler ^^^^^^^^^^^^^^^^^^^ -The :class:`BaseRotatingHandler` class, located in the :mod:`logging.handlers` +The :class:`BaseRotatingHandler` class, located in the :mod:`!logging.handlers` module, is the base class for the rotating file handlers, :class:`RotatingFileHandler` and :class:`TimedRotatingFileHandler`. You should not need to instantiate this class, but it has attributes and methods you may @@ -307,7 +304,7 @@ For an example, see :ref:`cookbook-rotator-namer`. RotatingFileHandler ^^^^^^^^^^^^^^^^^^^ -The :class:`RotatingFileHandler` class, located in the :mod:`logging.handlers` +The :class:`RotatingFileHandler` class, located in the :mod:`!logging.handlers` module, supports rotation of disk log files. @@ -362,7 +359,7 @@ TimedRotatingFileHandler ^^^^^^^^^^^^^^^^^^^^^^^^ The :class:`TimedRotatingFileHandler` class, located in the -:mod:`logging.handlers` module, supports rotation of disk log files at certain +:mod:`!logging.handlers` module, supports rotation of disk log files at certain timed intervals. @@ -463,6 +460,7 @@ timed intervals. .. method:: getFilesToDelete() Returns a list of filenames which should be deleted as part of rollover. These + are the absolute paths of the oldest backup log files written by the handler. .. method:: shouldRollover(record) @@ -474,7 +472,7 @@ timed intervals. SocketHandler ^^^^^^^^^^^^^ -The :class:`SocketHandler` class, located in the :mod:`logging.handlers` module, +The :class:`SocketHandler` class, located in the :mod:`!logging.handlers` module, sends logging output to a network socket. The base class uses a TCP socket. @@ -570,7 +568,7 @@ sends logging output to a network socket. The base class uses a TCP socket. DatagramHandler ^^^^^^^^^^^^^^^ -The :class:`DatagramHandler` class, located in the :mod:`logging.handlers` +The :class:`DatagramHandler` class, located in the :mod:`!logging.handlers` module, inherits from :class:`SocketHandler` to support sending logging messages over UDP sockets. @@ -617,7 +615,7 @@ over UDP sockets. SysLogHandler ^^^^^^^^^^^^^ -The :class:`SysLogHandler` class, located in the :mod:`logging.handlers` module, +The :class:`SysLogHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a remote or local Unix syslog. @@ -796,7 +794,7 @@ supports sending logging messages to a remote or local Unix syslog. NTEventLogHandler ^^^^^^^^^^^^^^^^^ -The :class:`NTEventLogHandler` class, located in the :mod:`logging.handlers` +The :class:`NTEventLogHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a local Windows NT, Windows 2000 or Windows XP event log. Before you can use it, you need Mark Hammond's Win32 extensions for Python installed. @@ -863,7 +861,7 @@ extensions for Python installed. SMTPHandler ^^^^^^^^^^^ -The :class:`SMTPHandler` class, located in the :mod:`logging.handlers` module, +The :class:`SMTPHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to an email address via SMTP. @@ -904,7 +902,7 @@ supports sending logging messages to an email address via SMTP. MemoryHandler ^^^^^^^^^^^^^ -The :class:`MemoryHandler` class, located in the :mod:`logging.handlers` module, +The :class:`MemoryHandler` class, located in the :mod:`!logging.handlers` module, supports buffering of logging records in memory, periodically flushing them to a :dfn:`target` handler. Flushing occurs whenever the buffer is full, or when an event of a certain severity or greater is seen. @@ -984,7 +982,7 @@ should, then :meth:`flush` is expected to do the flushing. HTTPHandler ^^^^^^^^^^^ -The :class:`HTTPHandler` class, located in the :mod:`logging.handlers` module, +The :class:`HTTPHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a web server, using either ``GET`` or ``POST`` semantics. @@ -1036,7 +1034,7 @@ QueueHandler .. versionadded:: 3.2 -The :class:`QueueHandler` class, located in the :mod:`logging.handlers` module, +The :class:`QueueHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a queue, such as those implemented in the :mod:`queue` or :mod:`multiprocessing` modules. @@ -1129,7 +1127,7 @@ QueueListener .. versionadded:: 3.2 -The :class:`QueueListener` class, located in the :mod:`logging.handlers` +The :class:`QueueListener` class, located in the :mod:`!logging.handlers` module, supports receiving logging messages from a queue, such as those implemented in the :mod:`queue` or :mod:`multiprocessing` modules. The messages are received from a queue in an internal thread and passed, on diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 425025931d9..aba530844d7 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -4,9 +4,6 @@ .. module:: logging :synopsis: Flexible event logging system for applications. -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - **Source code:** :source:`Lib/logging/__init__.py` .. index:: pair: Errors; logging @@ -1011,6 +1008,11 @@ the options available to you. | exc_info | You shouldn't need to | Exception tuple (à la ``sys.exc_info``) or, | | | format this yourself. | if no exception has occurred, ``None``. | +----------------+-------------------------+-----------------------------------------------+ +| exc_text | You shouldn't need to | Exception information formatted as a string. | +| | format this yourself. | This is set when :meth:`Formatter.format` is | +| | | invoked, or ``None`` if no exception has | +| | | occurred. | ++----------------+-------------------------+-----------------------------------------------+ | filename | ``%(filename)s`` | Filename portion of ``pathname``. | +----------------+-------------------------+-----------------------------------------------+ | funcName | ``%(funcName)s`` | Name of function containing the logging call. | @@ -1082,12 +1084,13 @@ LoggerAdapter Objects information into logging calls. For a usage example, see the section on :ref:`adding contextual information to your logging output `. -.. class:: LoggerAdapter(logger, extra, merge_extra=False) +.. class:: LoggerAdapter(logger, extra=None, merge_extra=False) Returns an instance of :class:`LoggerAdapter` initialized with an - underlying :class:`Logger` instance, a dict-like object (*extra*), and a - boolean (*merge_extra*) indicating whether or not the *extra* argument of - individual log calls should be merged with the :class:`LoggerAdapter` extra. + underlying :class:`Logger` instance, an optional dict-like object (*extra*), + and an optional boolean (*merge_extra*) indicating whether or not + the *extra* argument of individual log calls should be merged with + the :class:`LoggerAdapter` extra. The default behavior is to ignore the *extra* argument of individual log calls and only use the one of the :class:`LoggerAdapter` instance @@ -1127,9 +1130,13 @@ information into logging calls. For a usage example, see the section on Attribute :attr:`!manager` and method :meth:`!_log` were added, which delegate to the underlying logger and allow adapters to be nested. + .. versionchanged:: 3.10 + + The *extra* argument is now optional. + .. versionchanged:: 3.13 - The *merge_extra* argument was added. + The *merge_extra* parameter was added. Thread Safety @@ -1541,7 +1548,7 @@ Module-Level Attributes Integration with the warnings module ------------------------------------ -The :func:`captureWarnings` function can be used to integrate :mod:`logging` +The :func:`captureWarnings` function can be used to integrate :mod:`!logging` with the :mod:`warnings` module. .. function:: captureWarnings(capture) @@ -1572,7 +1579,7 @@ with the :mod:`warnings` module. library. `Original Python logging package `_ - This is the original source for the :mod:`logging` package. The version of the + This is the original source for the :mod:`!logging` package. The version of the package available from this site is suitable for use with Python 1.5.2, 2.1.x - and 2.2.x, which do not include the :mod:`logging` package in the standard + and 2.2.x, which do not include the :mod:`!logging` package in the standard library. diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 69f7cb8d48d..6cede00b218 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -4,9 +4,6 @@ .. module:: lzma :synopsis: A Python wrapper for the liblzma compression library. -.. moduleauthor:: Nadeem Vawda -.. sectionauthor:: Nadeem Vawda - .. versionadded:: 3.3 **Source code:** :source:`Lib/lzma.py` @@ -23,6 +20,8 @@ module. Note that :class:`LZMAFile` and :class:`bz2.BZ2File` are *not* thread-safe, so if you need to use a single :class:`LZMAFile` instance from multiple threads, it is necessary to protect it with a lock. +.. include:: ../includes/optional-module.rst + .. exception:: LZMAError diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index e8a96f29ea1..5b9741bdbca 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -4,9 +4,6 @@ .. module:: mailbox :synopsis: Manipulate mailboxes in various formats -.. moduleauthor:: Gregory K. Johnson -.. sectionauthor:: Gregory K. Johnson - **Source code:** :source:`Lib/mailbox.py` -------------- @@ -78,6 +75,14 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. message. Failing to lock the mailbox runs the risk of losing messages or corrupting the entire mailbox. + The :class:`!Mailbox` class supports the :keyword:`with` statement. When used + as a context manager, :class:`!Mailbox` calls :meth:`lock` when the context is entered, + returns the mailbox object as the context object, and at context end calls :meth:`close`, + thereby releasing the lock. + + .. versionchanged:: 3.15 + Support for the :keyword:`with` statement was added. + :class:`!Mailbox` instances have the following methods: @@ -917,7 +922,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. copied; furthermore, any format-specific information is converted insofar as possible if *message* is a :class:`!Message` instance. If *message* is a string, a byte string, - or a file, it should contain an :rfc:`2822`\ -compliant message, which is read + or a file, it should contain an :rfc:`5322`\ -compliant message, which is read and parsed. Files should be open in binary mode, but text mode files are accepted for backward compatibility. @@ -1025,7 +1030,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: remove_flag(flag) Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than + remove more than one flag at a time, *flag* may be a string of more than one character. If "info" contains experimental information rather than flags, the current "info" is not modified. @@ -1190,7 +1195,7 @@ When a :class:`!MaildirMessage` instance is created based upon a .. method:: remove_flag(flag) Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than + remove more than one flag at a time, *flag* may be a string of more than one character. When an :class:`!mboxMessage` instance is created based upon a @@ -1562,7 +1567,7 @@ When a :class:`!BabylMessage` instance is created based upon an .. method:: remove_flag(flag) Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than + remove more than one flag at a time, *flag* may be a string of more than one character. When an :class:`!MMDFMessage` instance is created based upon a @@ -1641,7 +1646,7 @@ The following exception classes are defined in the :mod:`!mailbox` module: .. exception:: Error() - The based class for all other module-specific exceptions. + The base class for all other module-specific exceptions. .. exception:: NoSuchMailboxError() @@ -1661,7 +1666,7 @@ The following exception classes are defined in the :mod:`!mailbox` module: Raised when some mailbox-related condition beyond the control of the program causes it to be unable to proceed, such as when failing to acquire a lock that - another program already holds a lock, or when a uniquely generated file name + another program already holds, or when a uniquely generated file name already exists. diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index e8e9071a5c9..4fe34f0a3a3 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -20,7 +20,7 @@ rarely does). [#]_ This is not a general "persistence" module. For general persistence and transfer of Python objects through RPC calls, see the modules :mod:`pickle` and -:mod:`shelve`. The :mod:`marshal` module exists mainly to support reading and +:mod:`shelve`. The :mod:`!marshal` module exists mainly to support reading and writing the "pseudo-compiled" code for Python modules of :file:`.pyc` files. Therefore, the Python maintainers reserve the right to modify the marshal format in backward incompatible ways should the need arise. @@ -34,7 +34,7 @@ supports a substantially wider range of objects than marshal. .. warning:: - The :mod:`marshal` module is not intended to be secure against erroneous or + The :mod:`!marshal` module is not intended to be secure against erroneous or maliciously constructed data. Never unmarshal data received from an untrusted or unauthenticated source. @@ -51,8 +51,9 @@ this module. The following types are supported: * Strings (:class:`str`) and :class:`bytes`. :term:`Bytes-like objects ` like :class:`bytearray` are marshalled as :class:`!bytes`. -* Containers: :class:`tuple`, :class:`list`, :class:`set`, :class:`frozenset`, - and (since :data:`version` 5), :class:`slice`. +* Containers: :class:`tuple`, :class:`list`, :class:`dict`, :class:`frozendict` + (since :data:`version` 6), :class:`set`, :class:`frozenset`, and + :class:`slice` (since :data:`version` 5). It should be understood that these are supported only if the values contained therein are themselves supported. Recursive containers are supported since :data:`version` 3. @@ -71,6 +72,10 @@ this module. The following types are supported: Added format version 5, which allows marshalling slices. +.. versionchanged:: 3.15 + + Added format version 6, which allows marshalling :class:`frozendict`. + The module defines these functions: @@ -173,6 +178,8 @@ In addition, the following constants are defined: 4 Python 3.4 Efficient representation of short strings ------- --------------- ---------------------------------------------------- 5 Python 3.14 Support for :class:`slice` objects + ------- --------------- ---------------------------------------------------- + 6 Python 3.15 Support for :class:`frozendict` objects ======= =============== ==================================================== diff --git a/Doc/library/math.integer.rst b/Doc/library/math.integer.rst new file mode 100644 index 00000000000..c3f34cdfd85 --- /dev/null +++ b/Doc/library/math.integer.rst @@ -0,0 +1,85 @@ +:mod:`!math.integer` --- integer-specific mathematics functions +=============================================================== + +.. module:: math.integer + :synopsis: Integer-specific mathematics functions. + +.. versionadded:: 3.15 + +-------------- + +This module provides access to the mathematical functions defined for integer arguments. +These functions accept integers and objects that implement the +:meth:`~object.__index__` method which is used to convert the object to an integer +number. + +The following functions are provided by this module. All return values are +computed exactly and are integers. + + +.. function:: comb(n, k, /) + + Return the number of ways to choose *k* items from *n* items without repetition + and without order. + + Evaluates to ``n! / (k! * (n - k)!)`` when ``k <= n`` and evaluates + to zero when ``k > n``. + + Also called the binomial coefficient because it is equivalent + to the coefficient of k-th term in polynomial expansion of + ``(1 + x)ⁿ``. + + Raises :exc:`ValueError` if either of the arguments are negative. + + +.. function:: factorial(n, /) + + Return factorial of the nonnegative integer *n*. + + +.. function:: gcd(*integers) + + Return the greatest common divisor of the specified integer arguments. + If any of the arguments is nonzero, then the returned value is the largest + positive integer that is a divisor of all arguments. If all arguments + are zero, then the returned value is ``0``. ``gcd()`` without arguments + returns ``0``. + + +.. function:: isqrt(n, /) + + Return the integer square root of the nonnegative integer *n*. This is the + floor of the exact square root of *n*, or equivalently the greatest integer + *a* such that *a*\ ² |nbsp| ≤ |nbsp| *n*. + + For some applications, it may be more convenient to have the least integer + *a* such that *n* |nbsp| ≤ |nbsp| *a*\ ², or in other words the ceiling of + the exact square root of *n*. For positive *n*, this can be computed using + ``a = 1 + isqrt(n - 1)``. + + + .. |nbsp| unicode:: 0xA0 + :trim: + + +.. function:: lcm(*integers) + + Return the least common multiple of the specified integer arguments. + If all arguments are nonzero, then the returned value is the smallest + positive integer that is a multiple of all arguments. If any of the arguments + is zero, then the returned value is ``0``. ``lcm()`` without arguments + returns ``1``. + + +.. function:: perm(n, k=None, /) + + Return the number of ways to choose *k* items from *n* items + without repetition and with order. + + Evaluates to ``n! / (n - k)!`` when ``k <= n`` and evaluates + to zero when ``k > n``. + + If *k* is not specified or is ``None``, then *k* defaults to *n* + and the function returns ``n!``. + + Raises :exc:`ValueError` if either of the arguments are negative. diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 55f2de07553..9cc8c5d6886 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -27,15 +27,6 @@ noted otherwise, all return values are floats. ==================================================== ============================================ -**Number-theoretic functions** --------------------------------------------------------------------------------------------------- -:func:`comb(n, k) ` Number of ways to choose *k* items from *n* items without repetition and without order -:func:`factorial(n) ` *n* factorial -:func:`gcd(*integers) ` Greatest common divisor of the integer arguments -:func:`isqrt(n) ` Integer square root of a nonnegative integer *n* -:func:`lcm(*integers) ` Least common multiple of the integer arguments -:func:`perm(n, k) ` Number of ways to choose *k* items from *n* items without repetition and with order - **Floating point arithmetic** -------------------------------------------------------------------------------------------------- :func:`ceil(x) ` Ceiling of *x*, the smallest integer greater than or equal to *x* @@ -126,92 +117,6 @@ noted otherwise, all return values are floats. ==================================================== ============================================ -Number-theoretic functions --------------------------- - -.. function:: comb(n, k) - - Return the number of ways to choose *k* items from *n* items without repetition - and without order. - - Evaluates to ``n! / (k! * (n - k)!)`` when ``k <= n`` and evaluates - to zero when ``k > n``. - - Also called the binomial coefficient because it is equivalent - to the coefficient of k-th term in polynomial expansion of - ``(1 + x)ⁿ``. - - Raises :exc:`TypeError` if either of the arguments are not integers. - Raises :exc:`ValueError` if either of the arguments are negative. - - .. versionadded:: 3.8 - - -.. function:: factorial(n) - - Return factorial of the nonnegative integer *n*. - - .. versionchanged:: 3.10 - Floats with integral values (like ``5.0``) are no longer accepted. - - -.. function:: gcd(*integers) - - Return the greatest common divisor of the specified integer arguments. - If any of the arguments is nonzero, then the returned value is the largest - positive integer that is a divisor of all arguments. If all arguments - are zero, then the returned value is ``0``. ``gcd()`` without arguments - returns ``0``. - - .. versionadded:: 3.5 - - .. versionchanged:: 3.9 - Added support for an arbitrary number of arguments. Formerly, only two - arguments were supported. - - -.. function:: isqrt(n) - - Return the integer square root of the nonnegative integer *n*. This is the - floor of the exact square root of *n*, or equivalently the greatest integer - *a* such that *a*\ ² |nbsp| ≤ |nbsp| *n*. - - For some applications, it may be more convenient to have the least integer - *a* such that *n* |nbsp| ≤ |nbsp| *a*\ ², or in other words the ceiling of - the exact square root of *n*. For positive *n*, this can be computed using - ``a = 1 + isqrt(n - 1)``. - - .. versionadded:: 3.8 - - -.. function:: lcm(*integers) - - Return the least common multiple of the specified integer arguments. - If all arguments are nonzero, then the returned value is the smallest - positive integer that is a multiple of all arguments. If any of the arguments - is zero, then the returned value is ``0``. ``lcm()`` without arguments - returns ``1``. - - .. versionadded:: 3.9 - - -.. function:: perm(n, k=None) - - Return the number of ways to choose *k* items from *n* items - without repetition and with order. - - Evaluates to ``n! / (n - k)!`` when ``k <= n`` and evaluates - to zero when ``k > n``. - - If *k* is not specified or is ``None``, then *k* defaults to *n* - and the function returns ``n!``. - - Raises :exc:`TypeError` if either of the arguments are not integers. - Raises :exc:`ValueError` if either of the arguments are negative. - - .. versionadded:: 3.8 - - Floating point arithmetic ------------------------- @@ -259,7 +164,7 @@ Floating point arithmetic is, :func:`!fmax` is not required to be sensitive to the sign of such operands (see Annex F of the C11 standard, §F.10.0.3 and §F.10.9.2). - .. versionadded:: next + .. versionadded:: 3.15 .. function:: fmin(x, y) @@ -271,7 +176,7 @@ Floating point arithmetic is, :func:`!fmin` is not required to be sensitive to the sign of such operands (see Annex F of the C11 standard, §F.10.0.3 and §F.10.9.3). - .. versionadded:: next + .. versionadded:: 3.15 .. function:: fmod(x, y) @@ -408,7 +313,7 @@ Floating point manipulation functions nonzero number that is not a subnormal (see :func:`issubnormal`). Return ``False`` otherwise. - .. versionadded:: next + .. versionadded:: 3.15 .. function:: issubnormal(x) @@ -417,7 +322,7 @@ Floating point manipulation functions nonzero number with a magnitude smaller than :data:`sys.float_info.min`. Return ``False`` otherwise. - .. versionadded:: next + .. versionadded:: 3.15 .. function:: isinf(x) @@ -464,7 +369,7 @@ Floating point manipulation functions This is useful to detect the sign bit of zeroes, infinities and NaNs. - .. versionadded:: next + .. versionadded:: 3.15 .. function:: ulp(x) @@ -601,7 +506,7 @@ Summation and product functions Roughly equivalent to:: - sqrt(sum((px - qx) ** 2.0 for px, qx in zip(p, q))) + sqrt(sum((px - qx) ** 2.0 for px, qx in zip(p, q, strict=True))) .. versionadded:: 3.8 @@ -812,6 +717,74 @@ Special functions .. versionadded:: 3.2 +Number-theoretic functions +-------------------------- + +For backward compatibility, the :mod:`!math` module provides also aliases of +the following functions from the :mod:`math.integer` module: + +.. list-table:: + + * - .. function:: comb(n, k) + :no-typesetting: + + :func:`comb(n, k) ` + - Number of ways to choose *k* items from *n* items without repetition + and without order + + * - .. function:: factorial(n) + :no-typesetting: + + :func:`factorial(n) ` + - *n* factorial + + * - .. function:: gcd(*integers) + :no-typesetting: + + :func:`gcd(*integers) ` + - Greatest common divisor of the integer arguments + + * - .. function:: isqrt(n) + :no-typesetting: + + :func:`isqrt(n) ` + - Integer square root of a nonnegative integer *n* + + * - .. function:: lcm(*integers) + :no-typesetting: + + :func:`lcm(*integers) ` + - Least common multiple of the integer arguments + + * - .. function:: perm(n, k) + :no-typesetting: + + :func:`perm(n, k) ` + - Number of ways to choose *k* items from *n* items without repetition + and with order + +.. versionadded:: 3.5 + The :func:`gcd` function. + +.. versionadded:: 3.8 + The :func:`comb`, :func:`perm` and :func:`isqrt` functions. + +.. versionadded:: 3.9 + The :func:`lcm` function. + +.. versionchanged:: 3.9 + Added support for an arbitrary number of arguments in the :func:`gcd` + function. + Formerly, only two arguments were supported. + +.. versionchanged:: 3.10 + Floats with integral values (like ``5.0``) are no longer accepted in the + :func:`factorial` function. + +.. soft-deprecated:: 3.15 + Use the :mod:`math.integer` functions instead of these aliases. + + Constants --------- @@ -872,7 +845,7 @@ Constants .. impl-detail:: - The :mod:`math` module consists mostly of thin wrappers around the platform C + The :mod:`!math` module consists mostly of thin wrappers around the platform C math library functions. Behavior in exceptional cases follows Annex F of the C99 standard where appropriate. The current implementation will raise :exc:`ValueError` for invalid operations like ``sqrt(-1.0)`` or ``log(0.0)`` @@ -894,5 +867,5 @@ Constants Module :mod:`cmath` Complex number versions of many of these functions. -.. |nbsp| unicode:: 0xA0 - :trim: + Module :mod:`math.integer` + Integer-specific mathematics functions. diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 13511b16a0e..0facacd50fd 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -4,15 +4,13 @@ .. module:: mimetypes :synopsis: Mapping of filename extensions to MIME types. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/mimetypes.py` .. index:: pair: MIME; content type -------------- -The :mod:`mimetypes` module converts between a filename or URL and the MIME type +The :mod:`!mimetypes` module converts between a filename or URL and the MIME type associated with the filename extension. Conversions are provided from filename to MIME type and from MIME type to filename extension; encodings are not supported for the latter conversion. @@ -56,8 +54,8 @@ the information :func:`init` sets up. .. versionchanged:: 3.8 Added support for *url* being a :term:`path-like object`. - .. deprecated:: 3.13 - Passing a file path instead of URL is :term:`soft deprecated`. + .. soft-deprecated:: 3.13 + Passing a file path instead of URL. Use :func:`guess_file_type` for this. @@ -196,7 +194,7 @@ MimeTypes objects The :class:`MimeTypes` class may be useful for applications which may want more than one MIME-type database; it provides an interface similar to the one of the -:mod:`mimetypes` module. +:mod:`!mimetypes` module. .. class:: MimeTypes(filenames=(), strict=True) diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index d82dda9e54b..774d05fdcaa 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -78,7 +78,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length This mode is useful to limit the number of open file handles. The original file can be renamed (but not deleted) after closing *fileno*. - .. versionchanged:: next + .. versionchanged:: 3.15 The *trackfd* parameter was added. .. audit-event:: mmap.__new__ fileno,length,access,offset mmap.mmap @@ -212,7 +212,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length Writable :term:`bytes-like object` is now accepted. - .. method:: flush([offset[, size]]) + .. method:: flush([offset[, size]], *, flags=MS_SYNC) Flushes changes made to the in-memory copy of a file back to disk. Without use of this call there is no guarantee that changes are written back before @@ -221,6 +221,12 @@ To map anonymous memory, -1 should be passed as the fileno along with the length whole extent of the mapping is flushed. *offset* must be a multiple of the :const:`PAGESIZE` or :const:`ALLOCATIONGRANULARITY`. + The *flags* parameter specifies the synchronization behavior. + *flags* must be one of the :ref:`MS_* constants ` available + on the system. + + On Windows, the *flags* parameter is ignored. + ``None`` is returned to indicate success. An exception is raised when the call failed. @@ -229,6 +235,15 @@ To map anonymous memory, -1 should be passed as the fileno along with the length on error under Windows. A zero value was returned on success; an exception was raised on error under Unix. + .. versionchanged:: 3.15 + Allow specifying *offset* without *size*. Previously, both *offset* + and *size* parameters were required together. Now *offset* can be + specified alone, and the flush operation will extend from *offset* + to the end of the mmap. + + .. versionchanged:: 3.15 + Added *flags* parameter to control synchronization behavior. + .. method:: madvise(option[, start[, length]]) @@ -322,13 +337,24 @@ To map anonymous memory, -1 should be passed as the fileno along with the length .. versionadded:: 3.13 + .. method:: set_name(name, /) + + Annotate the memory mapping with the given *name* for easier identification + in ``/proc//maps`` if the kernel supports the feature and :option:`-X dev <-X>` is passed + to Python or if Python is built in :ref:`debug mode `. + The length of *name* must not exceed 67 bytes including the ``'\0'`` terminator. + + .. availability:: Linux >= 5.17 (kernel built with ``CONFIG_ANON_VMA_NAME`` option) + + .. versionadded:: 3.15 + .. method:: size() Return the length of the file, which can be larger than the size of the memory-mapped area. For an anonymous mapping, return its size. - .. versionchanged:: next + .. versionchanged:: 3.15 Anonymous mappings are now supported on Unix. @@ -444,3 +470,22 @@ MAP_* Constants :data:`MAP_TPRO`, :data:`MAP_TRANSLATED_ALLOW_EXECUTE`, and :data:`MAP_UNIX03` constants. +.. _ms-constants: + +MS_* Constants +++++++++++++++ + +.. data:: MS_SYNC + MS_ASYNC + MS_INVALIDATE + + These flags control the synchronization behavior for :meth:`mmap.flush`: + + * :data:`MS_SYNC` - Synchronous flush: writes are scheduled and the call + blocks until they are physically written to storage. + * :data:`MS_ASYNC` - Asynchronous flush: writes are scheduled but the call + returns immediately without waiting for completion. + * :data:`MS_INVALIDATE` - Invalidate cached data: invalidates other mappings + of the same file so they can see the changes. + + .. versionadded:: 3.15 diff --git a/Doc/library/modulefinder.rst b/Doc/library/modulefinder.rst index 823d853f1ed..d26dcbd5f68 100644 --- a/Doc/library/modulefinder.rst +++ b/Doc/library/modulefinder.rst @@ -4,8 +4,6 @@ .. module:: modulefinder :synopsis: Find modules used by a script. -.. sectionauthor:: A.M. Kuchling - **Source code:** :source:`Lib/modulefinder.py` -------------- diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index 327cc3602b1..6b49c1a9ccd 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -2,10 +2,9 @@ =========================================================== .. module:: msvcrt - :platform: Windows :synopsis: Miscellaneous useful routines from the MS VC++ runtime. -.. sectionauthor:: Fred L. Drake, Jr. +**Source code:** :source:`PC/msvcrtmodule.c` -------------- @@ -22,6 +21,8 @@ api. The normal API deals only with ASCII characters and is of limited use for internationalized applications. The wide char API should be used where ever possible. +.. availability:: Windows. + .. versionchanged:: 3.3 Operations in this module now raise :exc:`OSError` where :exc:`IOError` was raised. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index d18ada3511d..187143d02cd 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -13,17 +13,16 @@ Introduction ------------ -:mod:`multiprocessing` is a package that supports spawning processes using an -API similar to the :mod:`threading` module. The :mod:`multiprocessing` package +:mod:`!multiprocessing` is a package that supports spawning processes using an +API similar to the :mod:`threading` module. The :mod:`!multiprocessing` package offers both local and remote concurrency, effectively side-stepping the :term:`Global Interpreter Lock ` by using subprocesses instead of threads. Due -to this, the :mod:`multiprocessing` module allows the programmer to fully +to this, the :mod:`!multiprocessing` module allows the programmer to fully leverage multiple processors on a given machine. It runs on both POSIX and Windows. -The :mod:`multiprocessing` module also introduces APIs which do not have -analogs in the :mod:`threading` module. A prime example of this is the +The :mod:`!multiprocessing` module also introduces the :class:`~multiprocessing.pool.Pool` object which offers a convenient means of parallelizing the execution of a function across multiple input values, distributing the input data across processes (data parallelism). The following @@ -44,6 +43,10 @@ will print to standard output :: [1, 4, 9] +The :mod:`!multiprocessing` module also introduces APIs which do not have +analogs in the :mod:`threading` module, like the ability to :meth:`terminate +`, :meth:`interrupt ` or :meth:`kill +` a running process. .. seealso:: @@ -58,7 +61,7 @@ will print to standard output :: The :class:`Process` class ^^^^^^^^^^^^^^^^^^^^^^^^^^ -In :mod:`multiprocessing`, processes are spawned by creating a :class:`Process` +In :mod:`!multiprocessing`, processes are spawned by creating a :class:`Process` object and then calling its :meth:`~Process.start` method. :class:`Process` follows the API of :class:`threading.Thread`. A trivial example of a multiprocess program is :: @@ -108,7 +111,7 @@ could lead to an :exc:`AttributeError` in the child process trying to locate the Contexts and start methods ^^^^^^^^^^^^^^^^^^^^^^^^^^ -Depending on the platform, :mod:`multiprocessing` supports three ways +Depending on the platform, :mod:`!multiprocessing` supports three ways to start a process. These *start methods* are .. _multiprocessing-start-method-spawn: @@ -237,7 +240,7 @@ processes for a different context. In particular, locks created using the *fork* context cannot be passed to processes started using the *spawn* or *forkserver* start methods. -Libraries using :mod:`multiprocessing` or +Libraries using :mod:`!multiprocessing` or :class:`~concurrent.futures.ProcessPoolExecutor` should be designed to allow their users to provide their own multiprocessing context. Using a specific context of your own within a library can lead to incompatibilities with the @@ -255,7 +258,7 @@ requires a specific start method. Exchanging objects between processes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`multiprocessing` supports two types of communication channel between +:mod:`!multiprocessing` supports two types of communication channel between processes: **Queues** @@ -276,7 +279,7 @@ processes: p.join() Queues are thread and process safe. - Any object put into a :mod:`~multiprocessing` queue will be serialized. + Any object put into a :mod:`!multiprocessing` queue will be serialized. **Pipes** @@ -310,7 +313,7 @@ processes: Synchronization between processes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`multiprocessing` contains equivalents of all the synchronization +:mod:`!multiprocessing` contains equivalents of all the synchronization primitives from :mod:`threading`. For instance one can use a lock to ensure that only one process prints to standard output at a time:: @@ -341,7 +344,7 @@ avoid using shared state as far as possible. This is particularly true when using multiple processes. However, if you really do need to use some shared data then -:mod:`multiprocessing` provides a couple of ways of doing so. +:mod:`!multiprocessing` provides a couple of ways of doing so. **Shared memory** @@ -515,9 +518,24 @@ process which created it. Reference --------- -The :mod:`multiprocessing` package mostly replicates the API of the +The :mod:`!multiprocessing` package mostly replicates the API of the :mod:`threading` module. +.. _global-start-method: + +Global start method +^^^^^^^^^^^^^^^^^^^ + +Python supports several ways to create and initialize a process. +The global start method sets the default mechanism for creating a process. + +Several multiprocessing functions and methods that may also instantiate +certain objects will implicitly set the global start method to the system's default, +if it hasn’t been set already. The global start method can only be set once. +If you need to change the start method from the system default, you must +proactively set the global start method before calling functions or methods, +or creating these objects. + :class:`Process` and exceptions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -686,7 +704,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the The process's authentication key (a byte string). - When :mod:`multiprocessing` is initialized the main process is assigned a + When :mod:`!multiprocessing` is initialized the main process is assigned a random string using :func:`os.urandom`. When a :class:`Process` object is created, it will inherit the @@ -787,7 +805,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the .. exception:: ProcessError - The base class of all :mod:`multiprocessing` exceptions. + The base class of all :mod:`!multiprocessing` exceptions. .. exception:: BufferTooShort @@ -827,19 +845,19 @@ If you use :class:`JoinableQueue` then you **must** call semaphore used to count the number of unfinished tasks may eventually overflow, raising an exception. -One difference from other Python queue implementations, is that :mod:`multiprocessing` +One difference from other Python queue implementations, is that :mod:`!multiprocessing` queues serializes all objects that are put into them using :mod:`pickle`. -The object return by the get method is a re-created object that does not share memory -with the original object. +The object returned by the get method is a re-created object that does not share +memory with the original object. Note that one can also create a shared queue by using a manager object -- see :ref:`multiprocessing-managers`. .. note:: - :mod:`multiprocessing` uses the usual :exc:`queue.Empty` and + :mod:`!multiprocessing` uses the usual :exc:`queue.Empty` and :exc:`queue.Full` exceptions to signal a timeout. They are not available in - the :mod:`multiprocessing` namespace so you need to import them from + the :mod:`!multiprocessing` namespace so you need to import them from :mod:`queue`. .. note:: @@ -887,7 +905,7 @@ For an example of the usage of queues for interprocess communication see :ref:`multiprocessing-examples`. -.. function:: Pipe([duplex]) +.. function:: Pipe(duplex=True) Returns a pair ``(conn1, conn2)`` of :class:`~multiprocessing.connection.Connection` objects representing the @@ -907,11 +925,15 @@ For an example of the usage of queues for interprocess communication see locks/semaphores. When a process first puts an item on the queue a feeder thread is started which transfers objects from a buffer into the pipe. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + The usual :exc:`queue.Empty` and :exc:`queue.Full` exceptions from the standard library's :mod:`queue` module are raised to signal timeouts. :class:`Queue` implements all the methods of :class:`queue.Queue` except for - :meth:`~queue.Queue.task_done` and :meth:`~queue.Queue.join`. + :meth:`~queue.Queue.task_done`, :meth:`~queue.Queue.join`, and + :meth:`~queue.Queue.shutdown`. .. method:: qsize() @@ -1022,6 +1044,9 @@ For an example of the usage of queues for interprocess communication see It is a simplified :class:`Queue` type, very close to a locked :class:`Pipe`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + .. method:: close() Close the queue: release internal resources. @@ -1052,6 +1077,9 @@ For an example of the usage of queues for interprocess communication see :class:`JoinableQueue`, a :class:`Queue` subclass, is a queue which additionally has :meth:`task_done` and :meth:`join` methods. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + .. method:: task_done() Indicate that a formerly enqueued task is complete. Used by queue @@ -1125,7 +1153,7 @@ Miscellaneous .. function:: freeze_support() - Add support for when a program which uses :mod:`multiprocessing` has been + Add support for when a program which uses :mod:`!multiprocessing` has been frozen to produce an executable. (Has been tested with **py2exe**, **PyInstaller** and **cx_Freeze**.) @@ -1161,11 +1189,11 @@ Miscellaneous .. function:: get_context(method=None) Return a context object which has the same attributes as the - :mod:`multiprocessing` module. + :mod:`!multiprocessing` module. If *method* is ``None`` then the default context is returned. Note that if - the global start method has not been set, this will set it to the - default method. + the global start method has not been set, this will set it to the system default + See :ref:`global-start-method` for more details. Otherwise *method* should be ``'fork'``, ``'spawn'``, ``'forkserver'``. :exc:`ValueError` is raised if the specified start method is not available. See :ref:`multiprocessing-start-methods`. @@ -1176,10 +1204,9 @@ Miscellaneous Return the name of start method used for starting processes. - If the global start method has not been set and *allow_none* is - ``False``, then the start method is set to the default and the name - is returned. If the start method has not been set and *allow_none* is - ``True`` then ``None`` is returned. + If the global start method is not set and *allow_none* is ``False``, the global start + method is set to the default, and its name is returned. See + :ref:`global-start-method` for more details. The return value can be ``'fork'``, ``'spawn'``, ``'forkserver'`` or ``None``. See :ref:`multiprocessing-start-methods`. @@ -1196,7 +1223,7 @@ Miscellaneous Set the path of the Python interpreter to use when starting a child process. (By default :data:`sys.executable` is used). Embedders will probably need to - do some thing like :: + do something like :: set_executable(os.path.join(sys.exec_prefix, 'pythonw.exe')) @@ -1208,22 +1235,32 @@ Miscellaneous .. versionchanged:: 3.11 Accepts a :term:`path-like object`. -.. function:: set_forkserver_preload(module_names) +.. function:: set_forkserver_preload(module_names, *, on_error='ignore') Set a list of module names for the forkserver main process to attempt to import so that their already imported state is inherited by forked - processes. Any :exc:`ImportError` when doing so is silently ignored. - This can be used as a performance enhancement to avoid repeated work - in every process. + processes. This can be used as a performance enhancement to avoid repeated + work in every process. For this to work, it must be called before the forkserver process has been launched (before creating a :class:`Pool` or starting a :class:`Process`). + The *on_error* parameter controls how :exc:`ImportError` exceptions during + module preloading are handled: ``"ignore"`` (default) silently ignores + failures, ``"warn"`` causes the forkserver subprocess to emit an + :exc:`ImportWarning` to stderr, and ``"fail"`` causes the forkserver + subprocess to exit with the exception traceback on stderr, making + subsequent process creation fail with :exc:`EOFError` or + :exc:`ConnectionError`. + Only meaningful when using the ``'forkserver'`` start method. See :ref:`multiprocessing-start-methods`. .. versionadded:: 3.4 + .. versionchanged:: 3.15 + Added the *on_error* parameter. + .. function:: set_start_method(method, force=False) Set the method which should be used to start child processes. @@ -1243,7 +1280,7 @@ Miscellaneous .. note:: - :mod:`multiprocessing` contains no analogues of + :mod:`!multiprocessing` contains no analogues of :func:`threading.active_count`, :func:`threading.enumerate`, :func:`threading.settrace`, :func:`threading.setprofile`, :class:`threading.Timer`, or :class:`threading.local`. @@ -1299,12 +1336,12 @@ Connection objects are usually created using Note that multiple connection objects may be polled at once by using :func:`multiprocessing.connection.wait`. - .. method:: send_bytes(buffer[, offset[, size]]) + .. method:: send_bytes(buf[, offset[, size]]) Send byte data from a :term:`bytes-like object` as a complete message. - If *offset* is given then data is read from that position in *buffer*. If - *size* is given then that many bytes will be read from buffer. Very large + If *offset* is given then data is read from that position in *buf*. If + *size* is given then that many bytes will be read from *buf*. Very large buffers (approximately 32 MiB+, though it depends on the OS) may raise a :exc:`ValueError` exception @@ -1324,18 +1361,18 @@ Connection objects are usually created using alias of :exc:`OSError`. - .. method:: recv_bytes_into(buffer[, offset]) + .. method:: recv_bytes_into(buf[, offset]) - Read into *buffer* a complete message of byte data sent from the other end + Read into *buf* a complete message of byte data sent from the other end of the connection and return the number of bytes in the message. Blocks until there is something to receive. Raises :exc:`EOFError` if there is nothing left to receive and the other end was closed. - *buffer* must be a writable :term:`bytes-like object`. If + *buf* must be a writable :term:`bytes-like object`. If *offset* is given then the message will be written into the buffer from that position. Offset must be a non-negative integer less than the - length of *buffer* (in bytes). + length of *buf* (in bytes). If the buffer is too short then a :exc:`BufferTooShort` exception is raised and the complete message is available as ``e.args[0]`` where ``e`` @@ -1406,6 +1443,9 @@ object -- see :ref:`multiprocessing-managers`. A barrier object: a clone of :class:`threading.Barrier`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + .. versionadded:: 3.3 .. class:: BoundedSemaphore([value]) @@ -1413,6 +1453,9 @@ object -- see :ref:`multiprocessing-managers`. A bounded semaphore object: a close analog of :class:`threading.BoundedSemaphore`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + A solitary difference from its close analog exists: its ``acquire`` method's first argument is named *block*, as is consistent with :meth:`Lock.acquire`. @@ -1431,7 +1474,10 @@ object -- see :ref:`multiprocessing-managers`. A condition variable: an alias for :class:`threading.Condition`. If *lock* is specified then it should be a :class:`Lock` or :class:`RLock` - object from :mod:`multiprocessing`. + object from :mod:`!multiprocessing`. + + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. .. versionchanged:: 3.3 The :meth:`~threading.Condition.wait_for` method was added. @@ -1440,6 +1486,8 @@ object -- see :ref:`multiprocessing-managers`. A clone of :class:`threading.Event`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. .. class:: Lock() @@ -1455,6 +1503,9 @@ object -- see :ref:`multiprocessing-managers`. instance of ``multiprocessing.synchronize.Lock`` initialized with a default context. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + :class:`Lock` supports the :term:`context manager` protocol and thus may be used in :keyword:`with` statements. @@ -1512,6 +1563,9 @@ object -- see :ref:`multiprocessing-managers`. instance of ``multiprocessing.synchronize.RLock`` initialized with a default context. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + :class:`RLock` supports the :term:`context manager` protocol and thus may be used in :keyword:`with` statements. @@ -1571,15 +1625,28 @@ object -- see :ref:`multiprocessing-managers`. A semaphore object: a close analog of :class:`threading.Semaphore`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + A solitary difference from its close analog exists: its ``acquire`` method's first argument is named *block*, as is consistent with :meth:`Lock.acquire`. + + .. method:: get_value() + + Return the current value of semaphore. + + Note that this may raise :exc:`NotImplementedError` on platforms like + macOS where ``sem_getvalue()`` is not implemented. + + .. method:: locked() Return a boolean indicating whether this object is locked right now. .. versionadded:: 3.14 + .. note:: On macOS, ``sem_timedwait`` is unsupported, so calling ``acquire()`` with @@ -1637,11 +1704,14 @@ inherited by child processes. value is actually a synchronized wrapper for the array. *typecode_or_type* determines the type of the elements of the returned array: - it is either a ctypes type or a one character typecode of the kind used by - the :mod:`array` module. If *size_or_initializer* is an integer, then it - determines the length of the array, and the array will be initially zeroed. - Otherwise, *size_or_initializer* is a sequence which is used to initialize - the array and whose length determines the length of the array. + it is either a :ref:`ctypes type ` or a one + character typecode of the kind used by the :mod:`array` module with the + exception of ``'w'``, which is not supported. In addition, the ``'c'`` + typecode is an alias for :class:`ctypes.c_char`. If *size_or_initializer* + is an integer, then it determines the length of the array, and the array + will be initially zeroed. Otherwise, *size_or_initializer* is a sequence + which is used to initialize the array and whose length determines the length + of the array. If *lock* is ``True`` (the default) then a new lock object is created to synchronize access to the value. If *lock* is a :class:`Lock` or @@ -1656,13 +1726,13 @@ inherited by child processes. attributes which allow one to use it to store and retrieve strings. -The :mod:`multiprocessing.sharedctypes` module -"""""""""""""""""""""""""""""""""""""""""""""" +The :mod:`!multiprocessing.sharedctypes` module +""""""""""""""""""""""""""""""""""""""""""""""" .. module:: multiprocessing.sharedctypes :synopsis: Allocate ctypes objects from shared memory. -The :mod:`multiprocessing.sharedctypes` module provides functions for allocating +The :mod:`!multiprocessing.sharedctypes` module provides functions for allocating :mod:`ctypes` objects from shared memory which can be inherited by child processes. @@ -1705,7 +1775,7 @@ processes. attributes which allow one to use it to store and retrieve strings -- see documentation for :mod:`ctypes`. -.. function:: Array(typecode_or_type, size_or_initializer, *, lock=True) +.. function:: Array(typecode_or_type, size_or_initializer, *, lock=True, ctx=None) The same as :func:`RawArray` except that depending on the value of *lock* a process-safe synchronization wrapper may be returned instead of a raw ctypes @@ -1719,9 +1789,13 @@ processes. automatically protected by a lock, so it will not necessarily be "process-safe". - Note that *lock* is a keyword-only argument. + *ctx* is a context object, or ``None`` (use the current context). If ``None``, + calling this may set the global start method. See + :ref:`global-start-method` for more details. -.. function:: Value(typecode_or_type, *args, lock=True) + Note that *lock* and *ctx* are keyword-only parameters. + +.. function:: Value(typecode_or_type, *args, lock=True, ctx=None) The same as :func:`RawValue` except that depending on the value of *lock* a process-safe synchronization wrapper may be returned instead of a raw ctypes @@ -1734,19 +1808,27 @@ processes. automatically protected by a lock, so it will not necessarily be "process-safe". - Note that *lock* is a keyword-only argument. + *ctx* is a context object, or ``None`` (use the current context). If ``None``, + calling this may set the global start method. See + :ref:`global-start-method` for more details. + + Note that *lock* and *ctx* are keyword-only parameters. .. function:: copy(obj) Return a ctypes object allocated from shared memory which is a copy of the ctypes object *obj*. -.. function:: synchronized(obj[, lock]) +.. function:: synchronized(obj, lock=None, ctx=None) Return a process-safe wrapper object for a ctypes object which uses *lock* to synchronize access. If *lock* is ``None`` (the default) then a :class:`multiprocessing.RLock` object is created automatically. + *ctx* is a context object, or ``None`` (use the current context). If ``None``, + calling this may set the global start method. See + :ref:`global-start-method` for more details. + A synchronized wrapper will have two methods in addition to those of the object it wraps: :meth:`get_obj` returns the wrapped object and :meth:`get_lock` returns the lock object used for synchronization. @@ -1864,8 +1946,9 @@ their parent process exits. The manager classes are defined in the *serializer* must be ``'pickle'`` (use :mod:`pickle` serialization) or ``'xmlrpclib'`` (use :mod:`xmlrpc.client` serialization). - *ctx* is a context object, or ``None`` (use the current context). See the - :func:`get_context` function. + *ctx* is a context object, or ``None`` (use the current context). If ``None``, + calling this may set the global start method. See + :ref:`global-start-method` for more details. *shutdown_timeout* is a timeout in seconds used to wait until the process used by the manager completes in the :meth:`shutdown` method. If the @@ -2249,7 +2332,7 @@ demonstrates a level of control over the synchronization. .. note:: - The proxy types in :mod:`multiprocessing` do nothing to support comparisons + The proxy types in :mod:`!multiprocessing` do nothing to support comparisons by value. So, for instance, we have: .. doctest:: @@ -2358,7 +2441,9 @@ with the :class:`Pool` class. the worker processes. Usually a pool is created using the function :func:`multiprocessing.Pool` or the :meth:`Pool` method of a context object. In both cases *context* is set - appropriately. + appropriately. If ``None``, calling this function will have the side effect + of setting the current global start method if it has not been set already. + See the :func:`get_context` function. Note that the methods of the pool object should only be called by the process which created the pool. @@ -2389,7 +2474,7 @@ with the :class:`Pool` class. duration of the Pool's work queue. A frequent pattern found in other systems (such as Apache, mod_wsgi, etc) to free resources held by workers is to allow a worker within a pool to complete only a set - amount of work before being exiting, being cleaned up and a new + amount of work before exiting, being cleaned up and a new process spawned to replace the old one. The *maxtasksperchild* argument to the :class:`Pool` exposes this ability to the end user. @@ -2574,7 +2659,7 @@ Usually message passing between processes is done using queues or by using :class:`~Connection` objects returned by :func:`~multiprocessing.Pipe`. -However, the :mod:`multiprocessing.connection` module allows some extra +However, the :mod:`!multiprocessing.connection` module allows some extra flexibility. It basically gives a high level message oriented API for dealing with sockets or Windows named pipes. It also has support for *digest authentication* using the :mod:`hmac` module, and for polling @@ -2832,6 +2917,16 @@ between themselves. Suitable authentication keys can also be generated by using :func:`os.urandom`. +This authentication protects :class:`Listener` and :func:`Client` connections, +which are reachable by address. It is not applied to the anonymous pipes +created by :func:`~multiprocessing.Pipe` or used internally by +:class:`~multiprocessing.Queue`. +:mod:`multiprocessing` treats all local processes running as the same user as +trusted; on most operating systems such processes can access each other's pipe +file descriptors regardless. Applications that require isolation between +processes of the same user must arrange it at the operating-system level -- +for example, by running workers under a different user account or in a sandbox. + Logging ^^^^^^^ @@ -2843,7 +2938,7 @@ handler type) for messages from different processes to get mixed up. .. currentmodule:: multiprocessing .. function:: get_logger() - Returns the logger used by :mod:`multiprocessing`. If necessary, a new one + Returns the logger used by :mod:`!multiprocessing`. If necessary, a new one will be created. When first created the logger has level :const:`logging.NOTSET` and no @@ -2881,18 +2976,18 @@ Below is an example session with logging turned on:: For a full table of logging levels, see the :mod:`logging` module. -The :mod:`multiprocessing.dummy` module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The :mod:`!multiprocessing.dummy` module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: multiprocessing.dummy :synopsis: Dumb wrapper around threading. -:mod:`multiprocessing.dummy` replicates the API of :mod:`multiprocessing` but is +:mod:`!multiprocessing.dummy` replicates the API of :mod:`!multiprocessing` but is no more than a wrapper around the :mod:`threading` module. .. currentmodule:: multiprocessing.pool -In particular, the ``Pool`` function provided by :mod:`multiprocessing.dummy` +In particular, the ``Pool`` function provided by :mod:`!multiprocessing.dummy` returns an instance of :class:`ThreadPool`, which is a subclass of :class:`Pool` that supports all the same method calls but uses a pool of worker threads rather than worker processes. @@ -2937,7 +3032,7 @@ Programming guidelines ---------------------- There are certain guidelines and idioms which should be adhered to when using -:mod:`multiprocessing`. +:mod:`!multiprocessing`. All start methods @@ -2978,7 +3073,7 @@ Joining zombie processes Better to inherit than pickle/unpickle When using the *spawn* or *forkserver* start methods many types - from :mod:`multiprocessing` need to be picklable so that child + from :mod:`!multiprocessing` need to be picklable so that child processes can use them. However, one should generally avoid sending shared objects to other processes using pipes or queues. Instead you should arrange the program so that a process which @@ -3068,7 +3163,7 @@ Explicitly pass resources to child processes Beware of replacing :data:`sys.stdin` with a "file like object" - :mod:`multiprocessing` originally unconditionally called:: + :mod:`!multiprocessing` originally unconditionally called:: os.close(sys.stdin.fileno()) diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index 74c97e8c9a9..3fbd2b57426 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -4,9 +4,6 @@ .. module:: netrc :synopsis: Loading of .netrc files. -.. moduleauthor:: Eric S. Raymond -.. sectionauthor:: Eric S. Raymond - **Source code:** :source:`Lib/netrc.py` -------------- diff --git a/Doc/library/numeric.rst b/Doc/library/numeric.rst index 7c76a479d73..a2109a29bfb 100644 --- a/Doc/library/numeric.rst +++ b/Doc/library/numeric.rst @@ -19,6 +19,7 @@ The following modules are documented in this chapter: numbers.rst math.rst + math.integer.rst cmath.rst decimal.rst fractions.rst diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index e8e71068dd9..c0dab83977e 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -4,8 +4,6 @@ .. module:: operator :synopsis: Functions corresponding to the standard operators. -.. sectionauthor:: Skip Montanaro - **Source code:** :source:`Lib/operator.py` .. testsetup:: @@ -15,7 +13,7 @@ -------------- -The :mod:`operator` module exports a set of efficient functions corresponding to +The :mod:`!operator` module exports a set of efficient functions corresponding to the intrinsic operators of Python. For example, ``operator.add(x, y)`` is equivalent to the expression ``x+y``. Many function names are those used for special methods, without the double underscores. For backward compatibility, @@ -275,7 +273,7 @@ The following operation works with callables: .. versionadded:: 3.11 -The :mod:`operator` module also defines tools for generalized attribute and item +The :mod:`!operator` module also defines tools for generalized attribute and item lookups. These are useful for making fast field extractors as arguments for :func:`map`, :func:`sorted`, :meth:`itertools.groupby`, or other functions that expect a function argument. @@ -390,7 +388,7 @@ Mapping Operators to Functions ------------------------------ This table shows how abstract operations correspond to operator symbols in the -Python syntax and the functions in the :mod:`operator` module. +Python syntax and the functions in the :mod:`!operator` module. +-----------------------+-------------------------+---------------------------------------+ | Operation | Syntax | Function | diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index ff327cf9162..905212965bd 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -4,9 +4,6 @@ .. module:: optparse :synopsis: Command-line option parsing library. -.. moduleauthor:: Greg Ward -.. sectionauthor:: Greg Ward - **Source code:** :source:`Lib/optparse.py` -------------- @@ -20,7 +17,7 @@ The standard library includes three argument parsing libraries: * :mod:`getopt`: a module that closely mirrors the procedural C ``getopt`` API. Included in the standard library since before the initial Python 1.0 release. -* :mod:`optparse`: a declarative replacement for ``getopt`` that +* :mod:`!optparse`: a declarative replacement for ``getopt`` that provides equivalent functionality without requiring each application to implement its own procedural option parsing logic. Included in the standard library since the Python 2.3 release. @@ -37,10 +34,10 @@ the highest level of baseline functionality with the least application level cod However, it also serves a niche use case as a tool for prototyping and testing command line argument handling in ``getopt``-based C applications. -:mod:`optparse` should be considered as an alternative to :mod:`argparse` in the +:mod:`!optparse` should be considered as an alternative to :mod:`argparse` in the following cases: -* an application is already using :mod:`optparse` and doesn't want to risk the +* an application is already using :mod:`!optparse` and doesn't want to risk the subtle behavioural changes that may arise when migrating to :mod:`argparse` * the application requires additional control over the way options and positional parameters are interleaved on the command line (including @@ -55,7 +52,7 @@ following cases: behavior which ``argparse`` does not support, but which can be implemented in terms of the lower level interface offered by ``optparse`` -These considerations also mean that :mod:`optparse` is likely to provide a +These considerations also mean that :mod:`!optparse` is likely to provide a better foundation for library authors writing third party command line argument processing libraries. @@ -126,15 +123,15 @@ application use case. Introduction ------------ -:mod:`optparse` is a more convenient, flexible, and powerful library for parsing +:mod:`!optparse` is a more convenient, flexible, and powerful library for parsing command-line options than the minimalist :mod:`getopt` module. -:mod:`optparse` uses a more declarative style of command-line parsing: +:mod:`!optparse` uses a more declarative style of command-line parsing: you create an instance of :class:`OptionParser`, populate it with options, and parse the command line. -:mod:`optparse` allows users to specify options in the conventional +:mod:`!optparse` allows users to specify options in the conventional GNU/POSIX syntax, and additionally generates usage and help messages for you. -Here's an example of using :mod:`optparse` in a simple script:: +Here's an example of using :mod:`!optparse` in a simple script:: from optparse import OptionParser ... @@ -152,11 +149,11 @@ on the command-line, for example:: --file=outfile -q -As it parses the command line, :mod:`optparse` sets attributes of the +As it parses the command line, :mod:`!optparse` sets attributes of the ``options`` object returned by :meth:`~OptionParser.parse_args` based on user-supplied command-line values. When :meth:`~OptionParser.parse_args` returns from parsing this command line, ``options.filename`` will be ``"outfile"`` and ``options.verbose`` will be -``False``. :mod:`optparse` supports both long and short options, allows short +``False``. :mod:`!optparse` supports both long and short options, allows short options to be merged together, and allows options to be associated with their arguments in a variety of ways. Thus, the following command lines are all equivalent to the above example:: @@ -171,7 +168,7 @@ Additionally, users can run one of the following :: -h --help -and :mod:`optparse` will print out a brief summary of your script's options: +and :mod:`!optparse` will print out a brief summary of your script's options: .. code-block:: text @@ -191,7 +188,7 @@ where the value of *yourscript* is determined at runtime (normally from Background ---------- -:mod:`optparse` was explicitly designed to encourage the creation of programs +:mod:`!optparse` was explicitly designed to encourage the creation of programs with straightforward command-line interfaces that follow the conventions established by the :c:func:`!getopt` family of functions available to C developers. To that end, it supports only the most common command-line syntax and semantics @@ -223,7 +220,7 @@ option options to be merged into a single argument, e.g. ``-x -F`` is equivalent to ``-xF``. The GNU project introduced ``--`` followed by a series of hyphen-separated words, e.g. ``--file`` or ``--dry-run``. These are the - only two option syntaxes provided by :mod:`optparse`. + only two option syntaxes provided by :mod:`!optparse`. Some other option syntaxes that the world has seen include: @@ -240,7 +237,7 @@ option * a slash followed by a letter, or a few letters, or a word, e.g. ``/f``, ``/file`` - These option syntaxes are not supported by :mod:`optparse`, and they never + These option syntaxes are not supported by :mod:`!optparse`, and they never will be. This is deliberate: the first three are non-standard on any environment, and the last only makes sense if you're exclusively targeting Windows or certain legacy platforms (e.g. VMS, MS-DOS). @@ -248,7 +245,7 @@ option option argument an argument that follows an option, is closely associated with that option, and is consumed from the argument list when that option is. With - :mod:`optparse`, option arguments may either be in a separate argument from + :mod:`!optparse`, option arguments may either be in a separate argument from their option: .. code-block:: text @@ -268,7 +265,7 @@ option argument will take an argument if they see it, and won't if they don't. This is somewhat controversial, because it makes parsing ambiguous: if ``-a`` takes an optional argument and ``-b`` is another option entirely, how do we - interpret ``-ab``? Because of this ambiguity, :mod:`optparse` does not + interpret ``-ab``? Because of this ambiguity, :mod:`!optparse` does not support this feature. positional argument @@ -278,7 +275,7 @@ positional argument required option an option that must be supplied on the command-line; note that the phrase - "required option" is self-contradictory in English. :mod:`optparse` doesn't + "required option" is self-contradictory in English. :mod:`!optparse` doesn't prevent you from implementing required options, but doesn't give you much help at it either. @@ -357,9 +354,9 @@ too many options can overwhelm users and make your code much harder to maintain. Tutorial -------- -While :mod:`optparse` is quite flexible and powerful, it's also straightforward +While :mod:`!optparse` is quite flexible and powerful, it's also straightforward to use in most cases. This section covers the code patterns that are common to -any :mod:`optparse`\ -based program. +any :mod:`!optparse`\ -based program. First, you need to import the OptionParser class; then, early in the main program, create an OptionParser instance:: @@ -374,7 +371,7 @@ Then you can start defining options. The basic syntax is:: attr=value, ...) Each option has one or more option strings, such as ``-f`` or ``--file``, -and several option attributes that tell :mod:`optparse` what to expect and what +and several option attributes that tell :mod:`!optparse` what to expect and what to do when it encounters that option on the command line. Typically, each option will have one short option string and one long option @@ -389,10 +386,10 @@ string overall. The option strings passed to :meth:`OptionParser.add_option` are effectively labels for the option defined by that call. For brevity, we will frequently refer to -*encountering an option* on the command line; in reality, :mod:`optparse` +*encountering an option* on the command line; in reality, :mod:`!optparse` encounters *option strings* and looks up options from them. -Once all of your options are defined, instruct :mod:`optparse` to parse your +Once all of your options are defined, instruct :mod:`!optparse` to parse your program's command line:: (options, args) = parser.parse_args() @@ -420,14 +417,14 @@ most fundamental. Understanding option actions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Actions tell :mod:`optparse` what to do when it encounters an option on the -command line. There is a fixed set of actions hard-coded into :mod:`optparse`; +Actions tell :mod:`!optparse` what to do when it encounters an option on the +command line. There is a fixed set of actions hard-coded into :mod:`!optparse`; adding new actions is an advanced topic covered in section -:ref:`optparse-extending-optparse`. Most actions tell :mod:`optparse` to store +:ref:`optparse-extending-optparse`. Most actions tell :mod:`!optparse` to store a value in some variable---for example, take a string from the command line and store it in an attribute of ``options``. -If you don't specify an option action, :mod:`optparse` defaults to ``store``. +If you don't specify an option action, :mod:`!optparse` defaults to ``store``. .. _optparse-store-action: @@ -435,7 +432,7 @@ If you don't specify an option action, :mod:`optparse` defaults to ``store``. The store action ^^^^^^^^^^^^^^^^ -The most common option action is ``store``, which tells :mod:`optparse` to take +The most common option action is ``store``, which tells :mod:`!optparse` to take the next argument (or the remainder of the current argument), ensure that it is of the correct type, and store it to your chosen destination. @@ -444,16 +441,16 @@ For example:: parser.add_option("-f", "--file", action="store", type="string", dest="filename") -Now let's make up a fake command line and ask :mod:`optparse` to parse it:: +Now let's make up a fake command line and ask :mod:`!optparse` to parse it:: args = ["-f", "foo.txt"] (options, args) = parser.parse_args(args) -When :mod:`optparse` sees the option string ``-f``, it consumes the next +When :mod:`!optparse` sees the option string ``-f``, it consumes the next argument, ``foo.txt``, and stores it in ``options.filename``. So, after this call to :meth:`~OptionParser.parse_args`, ``options.filename`` is ``"foo.txt"``. -Some other option types supported by :mod:`optparse` are ``int`` and ``float``. +Some other option types supported by :mod:`!optparse` are ``int`` and ``float``. Here's an option that expects an integer argument:: parser.add_option("-n", type="int", dest="num") @@ -470,19 +467,19 @@ right up against the option: since ``-n42`` (one argument) is equivalent to will print ``42``. -If you don't specify a type, :mod:`optparse` assumes ``string``. Combined with +If you don't specify a type, :mod:`!optparse` assumes ``string``. Combined with the fact that the default action is ``store``, that means our first example can be a lot shorter:: parser.add_option("-f", "--file", dest="filename") -If you don't supply a destination, :mod:`optparse` figures out a sensible +If you don't supply a destination, :mod:`!optparse` figures out a sensible default from the option strings: if the first long option string is ``--foo-bar``, then the default destination is ``foo_bar``. If there are no -long option strings, :mod:`optparse` looks at the first short option string: the +long option strings, :mod:`!optparse` looks at the first short option string: the default destination for ``-f`` is ``f``. -:mod:`optparse` also includes the built-in ``complex`` type. Adding +:mod:`!optparse` also includes the built-in ``complex`` type. Adding types is covered in section :ref:`optparse-extending-optparse`. @@ -492,7 +489,7 @@ Handling boolean (flag) options ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Flag options---set a variable to true or false when a particular option is -seen---are quite common. :mod:`optparse` supports them with two separate actions, +seen---are quite common. :mod:`!optparse` supports them with two separate actions, ``store_true`` and ``store_false``. For example, you might have a ``verbose`` flag that is turned on with ``-v`` and off with ``-q``:: @@ -503,7 +500,7 @@ Here we have two different options with the same destination, which is perfectly OK. (It just means you have to be a bit careful when setting default values---see below.) -When :mod:`optparse` encounters ``-v`` on the command line, it sets +When :mod:`!optparse` encounters ``-v`` on the command line, it sets ``options.verbose`` to ``True``; when it encounters ``-q``, ``options.verbose`` is set to ``False``. @@ -513,7 +510,7 @@ When :mod:`optparse` encounters ``-v`` on the command line, it sets Other actions ^^^^^^^^^^^^^ -Some other actions supported by :mod:`optparse` are: +Some other actions supported by :mod:`!optparse` are: ``"store_const"`` store a constant value, pre-set via :attr:`Option.const` @@ -539,11 +536,11 @@ Default values All of the above examples involve setting some variable (the "destination") when certain command-line options are seen. What happens if those options are never seen? Since we didn't supply any defaults, they are all set to ``None``. This -is usually fine, but sometimes you want more control. :mod:`optparse` lets you +is usually fine, but sometimes you want more control. :mod:`!optparse` lets you supply a default value for each destination, which is assigned before the command line is parsed. -First, consider the verbose/quiet example. If we want :mod:`optparse` to set +First, consider the verbose/quiet example. If we want :mod:`!optparse` to set ``verbose`` to ``True`` unless ``-q`` is seen, then we can do this:: parser.add_option("-v", action="store_true", dest="verbose", default=True) @@ -582,7 +579,7 @@ values, not both. Generating help ^^^^^^^^^^^^^^^ -:mod:`optparse`'s ability to generate help and usage text automatically is +:mod:`!optparse`'s ability to generate help and usage text automatically is useful for creating user-friendly command-line interfaces. All you have to do is supply a :attr:`~Option.help` value for each option, and optionally a short usage message for your whole program. Here's an OptionParser populated with @@ -603,7 +600,7 @@ user-friendly (documented) options:: help="interaction mode: novice, intermediate, " "or expert [default: %default]") -If :mod:`optparse` encounters either ``-h`` or ``--help`` on the +If :mod:`!optparse` encounters either ``-h`` or ``--help`` on the command-line, or if you just call :meth:`parser.print_help`, it prints the following to standard output: @@ -620,26 +617,26 @@ following to standard output: -m MODE, --mode=MODE interaction mode: novice, intermediate, or expert [default: intermediate] -(If the help output is triggered by a help option, :mod:`optparse` exits after +(If the help output is triggered by a help option, :mod:`!optparse` exits after printing the help text.) -There's a lot going on here to help :mod:`optparse` generate the best possible +There's a lot going on here to help :mod:`!optparse` generate the best possible help message: * the script defines its own usage message:: usage = "usage: %prog [options] arg1 arg2" - :mod:`optparse` expands ``%prog`` in the usage string to the name of the + :mod:`!optparse` expands ``%prog`` in the usage string to the name of the current program, i.e. ``os.path.basename(sys.argv[0])``. The expanded string is then printed before the detailed option help. - If you don't supply a usage string, :mod:`optparse` uses a bland but sensible + If you don't supply a usage string, :mod:`!optparse` uses a bland but sensible default: ``"Usage: %prog [options]"``, which is fine if your script doesn't take any positional arguments. * every option defines a help string, and doesn't worry about - line-wrapping---\ :mod:`optparse` takes care of wrapping lines and making + line-wrapping---\ :mod:`!optparse` takes care of wrapping lines and making the help output look good. * options that take a value indicate this fact in their automatically generated @@ -649,7 +646,7 @@ help message: Here, "MODE" is called the meta-variable: it stands for the argument that the user is expected to supply to ``-m``/``--mode``. By default, - :mod:`optparse` converts the destination variable name to uppercase and uses + :mod:`!optparse` converts the destination variable name to uppercase and uses that for the meta-variable. Sometimes, that's not what you want---for example, the ``--filename`` option explicitly sets ``metavar="FILE"``, resulting in this automatically generated option description:: @@ -663,7 +660,7 @@ help message: way to make your help text a lot clearer and more useful for end users. * options that have a default value can include ``%default`` in the help - string---\ :mod:`optparse` will replace it with :func:`str` of the option's + string---\ :mod:`!optparse` will replace it with :func:`str` of the option's default value. If an option has no default value (or the default value is ``None``), ``%default`` expands to ``none``. @@ -779,14 +776,14 @@ option groups is: Printing a version string ^^^^^^^^^^^^^^^^^^^^^^^^^ -Similar to the brief usage string, :mod:`optparse` can also print a version +Similar to the brief usage string, :mod:`!optparse` can also print a version string for your program. You have to supply the string as the ``version`` argument to OptionParser:: parser = OptionParser(usage="%prog [-f] [-q]", version="%prog 1.0") ``%prog`` is expanded just like it is in ``usage``. Apart from that, -``version`` can contain anything you like. When you supply it, :mod:`optparse` +``version`` can contain anything you like. When you supply it, :mod:`!optparse` automatically adds a ``--version`` option to your parser. If it encounters this option on the command line, it expands your ``version`` string (by replacing ``%prog``), prints it to stdout, and exits. @@ -815,10 +812,10 @@ The following two methods can be used to print and get the ``version`` string: .. _optparse-how-optparse-handles-errors: -How :mod:`optparse` handles errors -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +How :mod:`!optparse` handles errors +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -There are two broad classes of errors that :mod:`optparse` has to worry about: +There are two broad classes of errors that :mod:`!optparse` has to worry about: programmer errors and user errors. Programmer errors are usually erroneous calls to :func:`OptionParser.add_option`, e.g. invalid option strings, unknown option attributes, missing option attributes, etc. These are dealt with in the @@ -826,7 +823,7 @@ usual way: raise an exception (either :exc:`optparse.OptionError` or :exc:`TypeError`) and let the program crash. Handling user errors is much more important, since they are guaranteed to happen -no matter how stable your code is. :mod:`optparse` can automatically detect +no matter how stable your code is. :mod:`!optparse` can automatically detect some user errors, such as bad option arguments (passing ``-n 4x`` where ``-n`` takes an integer argument), missing arguments (``-n`` at the end of the command line, where ``-n`` takes an argument of any type). Also, @@ -838,7 +835,7 @@ condition:: if options.a and options.b: parser.error("options -a and -b are mutually exclusive") -In either case, :mod:`optparse` handles the error the same way: it prints the +In either case, :mod:`!optparse` handles the error the same way: it prints the program's usage message and an error message to standard error and exits with error status 2. @@ -861,11 +858,11 @@ Or, where the user fails to pass a value at all: foo: error: -n option requires an argument -:mod:`optparse`\ -generated error messages take care always to mention the +:mod:`!optparse`\ -generated error messages take care always to mention the option involved in the error; be sure to do the same when calling :func:`OptionParser.error` from your application code. -If :mod:`optparse`'s default error-handling behaviour does not suit your needs, +If :mod:`!optparse`'s default error-handling behaviour does not suit your needs, you'll need to subclass OptionParser and override its :meth:`~OptionParser.exit` and/or :meth:`~OptionParser.error` methods. @@ -875,7 +872,7 @@ and/or :meth:`~OptionParser.error` methods. Putting it all together ^^^^^^^^^^^^^^^^^^^^^^^ -Here's what :mod:`optparse`\ -based scripts usually look like:: +Here's what :mod:`!optparse`\ -based scripts usually look like:: from optparse import OptionParser ... @@ -911,7 +908,7 @@ Reference Guide Creating the parser ^^^^^^^^^^^^^^^^^^^ -The first step in using :mod:`optparse` is to create an OptionParser instance. +The first step in using :mod:`!optparse` is to create an OptionParser instance. .. class:: OptionParser(...) @@ -921,7 +918,7 @@ The first step in using :mod:`optparse` is to create an OptionParser instance. ``usage`` (default: ``"%prog [options]"``) The usage summary to print when your program is run incorrectly or with a - help option. When :mod:`optparse` prints the usage string, it expands + help option. When :mod:`!optparse` prints the usage string, it expands ``%prog`` to ``os.path.basename(sys.argv[0])`` (or to ``prog`` if you passed that keyword argument). To suppress a usage message, pass the special value :const:`optparse.SUPPRESS_USAGE`. @@ -938,7 +935,7 @@ The first step in using :mod:`optparse` is to create an OptionParser instance. ``version`` (default: ``None``) A version string to print when the user supplies a version option. If you - supply a true value for ``version``, :mod:`optparse` automatically adds a + supply a true value for ``version``, :mod:`!optparse` automatically adds a version option with the single option string ``--version``. The substring ``%prog`` is expanded the same as for ``usage``. @@ -949,17 +946,17 @@ The first step in using :mod:`optparse` is to create an OptionParser instance. ``description`` (default: ``None``) A paragraph of text giving a brief overview of your program. - :mod:`optparse` reformats this paragraph to fit the current terminal width + :mod:`!optparse` reformats this paragraph to fit the current terminal width and prints it when the user requests help (after ``usage``, but before the list of options). ``formatter`` (default: a new :class:`IndentedHelpFormatter`) An instance of optparse.HelpFormatter that will be used for printing help - text. :mod:`optparse` provides two concrete classes for this purpose: + text. :mod:`!optparse` provides two concrete classes for this purpose: IndentedHelpFormatter and TitledHelpFormatter. ``add_help_option`` (default: ``True``) - If true, :mod:`optparse` will add a help option (with option strings ``-h`` + If true, :mod:`!optparse` will add a help option (with option strings ``-h`` and ``--help``) to the parser. ``prog`` @@ -997,7 +994,7 @@ the OptionParser constructor, as in:: (:func:`make_option` is a factory function for creating Option instances; currently it is an alias for the Option constructor. A future version of -:mod:`optparse` may split Option into several classes, and :func:`make_option` +:mod:`!optparse` may split Option into several classes, and :func:`make_option` will pick the right class to instantiate. Do not instantiate Option directly.) @@ -1027,12 +1024,12 @@ The canonical way to create an :class:`Option` instance is with the The keyword arguments define attributes of the new Option object. The most important option attribute is :attr:`~Option.action`, and it largely determines which other attributes are relevant or required. If you pass - irrelevant option attributes, or fail to pass required ones, :mod:`optparse` + irrelevant option attributes, or fail to pass required ones, :mod:`!optparse` raises an :exc:`OptionError` exception explaining your mistake. - An option's *action* determines what :mod:`optparse` does when it encounters + An option's *action* determines what :mod:`!optparse` does when it encounters this option on the command-line. The standard option actions hard-coded into - :mod:`optparse` are: + :mod:`!optparse` are: ``"store"`` store this option's argument (default) @@ -1066,7 +1063,7 @@ The canonical way to create an :class:`Option` instance is with the attributes; see :ref:`optparse-standard-option-actions`.) As you can see, most actions involve storing or updating a value somewhere. -:mod:`optparse` always creates a special object for this, conventionally called +:mod:`!optparse` always creates a special object for this, conventionally called ``options``, which is an instance of :class:`optparse.Values`. .. class:: Values @@ -1084,7 +1081,7 @@ For example, when you call :: parser.parse_args() -one of the first things :mod:`optparse` does is create the ``options`` object:: +one of the first things :mod:`!optparse` does is create the ``options`` object:: options = Values() @@ -1099,7 +1096,7 @@ and the command-line being parsed includes any of the following:: --file=foo --file foo -then :mod:`optparse`, on seeing this option, will do the equivalent of :: +then :mod:`!optparse`, on seeing this option, will do the equivalent of :: options.filename = "foo" @@ -1124,13 +1121,13 @@ Option attributes The following option attributes may be passed as keyword arguments to :meth:`OptionParser.add_option`. If you pass an option attribute that is not relevant to a particular option, or fail to pass a required option attribute, -:mod:`optparse` raises :exc:`OptionError`. +:mod:`!optparse` raises :exc:`OptionError`. .. attribute:: Option.action (default: ``"store"``) - Determines :mod:`optparse`'s behaviour when this option is seen on the + Determines :mod:`!optparse`'s behaviour when this option is seen on the command line; the available options are documented :ref:`here `. @@ -1147,8 +1144,8 @@ relevant to a particular option, or fail to pass a required option attribute, (default: derived from option strings) If the option's action implies writing or modifying a value somewhere, this - tells :mod:`optparse` where to write it: :attr:`~Option.dest` names an - attribute of the ``options`` object that :mod:`optparse` builds as it parses + tells :mod:`!optparse` where to write it: :attr:`~Option.dest` names an + attribute of the ``options`` object that :mod:`!optparse` builds as it parses the command line. .. attribute:: Option.default @@ -1161,7 +1158,7 @@ relevant to a particular option, or fail to pass a required option attribute, (default: 1) How many arguments of type :attr:`~Option.type` should be consumed when this - option is seen. If > 1, :mod:`optparse` will store a tuple of values to + option is seen. If > 1, :mod:`!optparse` will store a tuple of values to :attr:`~Option.dest`. .. attribute:: Option.const @@ -1207,7 +1204,7 @@ Standard option actions The various option actions all have slightly different requirements and effects. Most actions have several relevant option attributes which you may specify to -guide :mod:`optparse`'s behaviour; a few have required attributes, which you +guide :mod:`!optparse`'s behaviour; a few have required attributes, which you must specify for any option using that action. * ``"store"`` [relevant: :attr:`~Option.type`, :attr:`~Option.dest`, @@ -1225,9 +1222,9 @@ must specify for any option using that action. If :attr:`~Option.type` is not supplied, it defaults to ``"string"``. - If :attr:`~Option.dest` is not supplied, :mod:`optparse` derives a destination + If :attr:`~Option.dest` is not supplied, :mod:`!optparse` derives a destination from the first long option string (e.g., ``--foo-bar`` implies - ``foo_bar``). If there are no long option strings, :mod:`optparse` derives a + ``foo_bar``). If there are no long option strings, :mod:`!optparse` derives a destination from the first short option string (e.g., ``-f`` implies ``f``). Example:: @@ -1239,7 +1236,7 @@ must specify for any option using that action. -f foo.txt -p 1 -3.5 4 -fbar.txt - :mod:`optparse` will set :: + :mod:`!optparse` will set :: options.f = "foo.txt" options.point = (1.0, -3.5, 4.0) @@ -1259,7 +1256,7 @@ must specify for any option using that action. parser.add_option("--noisy", action="store_const", const=2, dest="verbose") - If ``--noisy`` is seen, :mod:`optparse` will set :: + If ``--noisy`` is seen, :mod:`!optparse` will set :: options.verbose = 2 @@ -1282,7 +1279,7 @@ must specify for any option using that action. The option must be followed by an argument, which is appended to the list in :attr:`~Option.dest`. If no default value for :attr:`~Option.dest` is - supplied, an empty list is automatically created when :mod:`optparse` first + supplied, an empty list is automatically created when :mod:`!optparse` first encounters this option on the command-line. If :attr:`~Option.nargs` > 1, multiple arguments are consumed, and a tuple of length :attr:`~Option.nargs` is appended to :attr:`~Option.dest`. @@ -1294,7 +1291,7 @@ must specify for any option using that action. parser.add_option("-t", "--tracks", action="append", type="int") - If ``-t3`` is seen on the command-line, :mod:`optparse` does the equivalent + If ``-t3`` is seen on the command-line, :mod:`!optparse` does the equivalent of:: options.tracks = [] @@ -1333,7 +1330,7 @@ must specify for any option using that action. parser.add_option("-v", action="count", dest="verbosity") - The first time ``-v`` is seen on the command line, :mod:`optparse` does the + The first time ``-v`` is seen on the command line, :mod:`!optparse` does the equivalent of:: options.verbosity = 0 @@ -1364,7 +1361,7 @@ must specify for any option using that action. listed in the help message. To omit an option entirely, use the special value :const:`optparse.SUPPRESS_HELP`. - :mod:`optparse` automatically adds a :attr:`~Option.help` option to all + :mod:`!optparse` automatically adds a :attr:`~Option.help` option to all OptionParsers, so you do not normally need to create one. Example:: @@ -1382,7 +1379,7 @@ must specify for any option using that action. help="Input file to read data from") parser.add_option("--secret", help=SUPPRESS_HELP) - If :mod:`optparse` sees either ``-h`` or ``--help`` on the command line, + If :mod:`!optparse` sees either ``-h`` or ``--help`` on the command line, it will print something like the following help message to stdout (assuming ``sys.argv[0]`` is ``"foo.py"``): @@ -1395,7 +1392,7 @@ must specify for any option using that action. -v Be moderately verbose --file=FILENAME Input file to read data from - After printing the help message, :mod:`optparse` terminates your process with + After printing the help message, :mod:`!optparse` terminates your process with ``sys.exit(0)``. * ``"version"`` @@ -1405,7 +1402,7 @@ must specify for any option using that action. ``print_version()`` method of OptionParser. Generally only relevant if the ``version`` argument is supplied to the OptionParser constructor. As with :attr:`~Option.help` options, you will rarely create ``version`` options, - since :mod:`optparse` automatically adds them when needed. + since :mod:`!optparse` automatically adds them when needed. .. _optparse-standard-option-types: @@ -1413,7 +1410,7 @@ must specify for any option using that action. Standard option types ^^^^^^^^^^^^^^^^^^^^^ -:mod:`optparse` has five built-in option types: ``"string"``, ``"int"``, +:mod:`!optparse` has five built-in option types: ``"string"``, ``"int"``, ``"choice"``, ``"float"`` and ``"complex"``. If you need to add new option types, see section :ref:`optparse-extending-optparse`. @@ -1432,7 +1429,7 @@ Integer arguments (type ``"int"``) are parsed as follows: The conversion is done by calling :func:`int` with the appropriate base (2, 8, -10, or 16). If this fails, so will :mod:`optparse`, although with a more useful +10, or 16). If this fails, so will :mod:`!optparse`, although with a more useful error message. ``"float"`` and ``"complex"`` option arguments are converted directly with @@ -1471,7 +1468,7 @@ The whole point of creating and populating an OptionParser is to call its ``options`` the same object that was passed in as *values*, or the ``optparse.Values`` - instance created by :mod:`optparse` + instance created by :mod:`!optparse` ``args`` the leftover positional arguments after all options have been processed @@ -1499,7 +1496,7 @@ provides several methods to help you out: .. method:: OptionParser.disable_interspersed_args() Set parsing to stop on the first non-option. For example, if ``-a`` and - ``-b`` are both simple options that take no arguments, :mod:`optparse` + ``-b`` are both simple options that take no arguments, :mod:`!optparse` normally accepts this syntax:: prog -a arg1 -b arg2 @@ -1554,7 +1551,7 @@ strings:: (This is particularly true if you've defined your own OptionParser subclass with some standard options.) -Every time you add an option, :mod:`optparse` checks for conflicts with existing +Every time you add an option, :mod:`!optparse` checks for conflicts with existing options. If it finds any, it invokes the current conflict-handling mechanism. You can set the conflict-handling mechanism either in the constructor:: @@ -1581,7 +1578,7 @@ intelligently and add conflicting options to it:: parser.add_option("-n", "--dry-run", ..., help="do no harm") parser.add_option("-n", "--noisy", ..., help="be noisy") -At this point, :mod:`optparse` detects that a previously added option is already +At this point, :mod:`!optparse` detects that a previously added option is already using the ``-n`` option string. Since ``conflict_handler`` is ``"resolve"``, it resolves the situation by removing ``-n`` from the earlier option's list of option strings. Now ``--dry-run`` is the only way for the user to activate @@ -1594,14 +1591,14 @@ that option. If the user asks for help, the help message will reflect that:: It's possible to whittle away the option strings for a previously added option until there are none left, and the user has no way of invoking that option from -the command-line. In that case, :mod:`optparse` removes that option completely, +the command-line. In that case, :mod:`!optparse` removes that option completely, so it doesn't show up in help text or anywhere else. Carrying on with our existing OptionParser:: parser.add_option("--dry-run", ..., help="new dry-run option") At this point, the original ``-n``/``--dry-run`` option is no longer -accessible, so :mod:`optparse` removes it, leaving this help text:: +accessible, so :mod:`!optparse` removes it, leaving this help text:: Options: ... @@ -1676,9 +1673,9 @@ OptionParser supports several other public methods: Option Callbacks ---------------- -When :mod:`optparse`'s built-in actions and types aren't quite enough for your -needs, you have two choices: extend :mod:`optparse` or define a callback option. -Extending :mod:`optparse` is more general, but overkill for a lot of simple +When :mod:`!optparse`'s built-in actions and types aren't quite enough for your +needs, you have two choices: extend :mod:`!optparse` or define a callback option. +Extending :mod:`!optparse` is more general, but overkill for a lot of simple cases. Quite often a simple callback is all you need. There are two steps to defining a callback option: @@ -1702,14 +1699,14 @@ only option attribute you must specify is ``callback``, the function to call:: ``callback`` is a function (or other callable object), so you must have already defined ``my_callback()`` when you create this callback option. In this simple -case, :mod:`optparse` doesn't even know if ``-c`` takes any arguments, +case, :mod:`!optparse` doesn't even know if ``-c`` takes any arguments, which usually means that the option takes no arguments---the mere presence of ``-c`` on the command-line is all it needs to know. In some circumstances, though, you might want your callback to consume an arbitrary number of command-line arguments. This is where writing callbacks gets tricky; it's covered later in this section. -:mod:`optparse` always passes four particular arguments to your callback, and it +:mod:`!optparse` always passes four particular arguments to your callback, and it will only pass additional arguments if you specify them via :attr:`~Option.callback_args` and :attr:`~Option.callback_kwargs`. Thus, the minimal callback function signature is:: @@ -1723,12 +1720,12 @@ callback option: :attr:`~Option.type` has its usual meaning: as with the ``"store"`` or ``"append"`` actions, it - instructs :mod:`optparse` to consume one argument and convert it to + instructs :mod:`!optparse` to consume one argument and convert it to :attr:`~Option.type`. Rather than storing the converted value(s) anywhere, - though, :mod:`optparse` passes it to your callback function. + though, :mod:`!optparse` passes it to your callback function. :attr:`~Option.nargs` - also has its usual meaning: if it is supplied and > 1, :mod:`optparse` will + also has its usual meaning: if it is supplied and > 1, :mod:`!optparse` will consume :attr:`~Option.nargs` arguments, each of which must be convertible to :attr:`~Option.type`. It then passes a tuple of converted values to your callback. @@ -1762,7 +1759,7 @@ where ``"--foobar"``.) ``value`` - is the argument to this option seen on the command-line. :mod:`optparse` will + is the argument to this option seen on the command-line. :mod:`!optparse` will only expect an argument if :attr:`~Option.type` is set; the type of ``value`` will be the type implied by the option's type. If :attr:`~Option.type` for this option is ``None`` (no argument expected), then ``value`` will be ``None``. If :attr:`~Option.nargs` @@ -1787,7 +1784,7 @@ where ``parser.values`` the object where option values are by default stored (an instance of optparse.OptionValues). This lets callbacks use the same mechanism as the - rest of :mod:`optparse` for storing option values; you don't need to mess + rest of :mod:`!optparse` for storing option values; you don't need to mess around with globals or closures. You can also access or modify the value(s) of any options already encountered on the command-line. @@ -1806,7 +1803,7 @@ Raising errors in a callback ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The callback function should raise :exc:`OptionValueError` if there are any -problems with the option or its argument(s). :mod:`optparse` catches this and +problems with the option or its argument(s). :mod:`!optparse` catches this and terminates the program, printing the error message you supply to stderr. Your message should be clear, concise, accurate, and mention the option at fault. Otherwise, the user will have a hard time figuring out what they did wrong. @@ -1906,7 +1903,7 @@ Here's an example that just emulates the standard ``"store"`` action:: action="callback", callback=store_value, type="int", nargs=3, dest="foo") -Note that :mod:`optparse` takes care of consuming 3 arguments and converting +Note that :mod:`!optparse` takes care of consuming 3 arguments and converting them to integers for you; all you have to do is store them. (Or whatever; obviously you don't need a callback for this example.) @@ -1917,9 +1914,9 @@ Callback example 6: variable arguments ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Things get hairy when you want an option to take a variable number of arguments. -For this case, you must write a callback, as :mod:`optparse` doesn't provide any +For this case, you must write a callback, as :mod:`!optparse` doesn't provide any built-in capabilities for it. And you have to deal with certain intricacies of -conventional Unix command-line parsing that :mod:`optparse` normally handles for +conventional Unix command-line parsing that :mod:`!optparse` normally handles for you. In particular, callbacks should implement the conventional rules for bare ``--`` and ``-`` arguments: @@ -1934,7 +1931,7 @@ you. In particular, callbacks should implement the conventional rules for bare If you want an option that takes a variable number of arguments, there are several subtle, tricky issues to worry about. The exact implementation you choose will be based on which trade-offs you're willing to make for your -application (which is why :mod:`optparse` doesn't support this sort of thing +application (which is why :mod:`!optparse` doesn't support this sort of thing directly). Nevertheless, here's a stab at a callback for an option with variable @@ -1970,10 +1967,10 @@ arguments:: .. _optparse-extending-optparse: -Extending :mod:`optparse` -------------------------- +Extending :mod:`!optparse` +-------------------------- -Since the two major controlling factors in how :mod:`optparse` interprets +Since the two major controlling factors in how :mod:`!optparse` interprets command-line options are the action and type of each option, the most likely direction of extension is to add new actions and new types. @@ -1983,9 +1980,9 @@ direction of extension is to add new actions and new types. Adding new types ^^^^^^^^^^^^^^^^ -To add new types, you need to define your own subclass of :mod:`optparse`'s +To add new types, you need to define your own subclass of :mod:`!optparse`'s :class:`Option` class. This class has a couple of attributes that define -:mod:`optparse`'s types: :attr:`~Option.TYPES` and :attr:`~Option.TYPE_CHECKER`. +:mod:`!optparse`'s types: :attr:`~Option.TYPES` and :attr:`~Option.TYPE_CHECKER`. .. attribute:: Option.TYPES @@ -2015,7 +2012,7 @@ To add new types, you need to define your own subclass of :mod:`optparse`'s Here's a silly example that demonstrates adding a ``"complex"`` option type to parse Python-style complex numbers on the command line. (This is even sillier -than it used to be, because :mod:`optparse` 1.3 added built-in support for +than it used to be, because :mod:`!optparse` 1.3 added built-in support for complex numbers, but never mind.) First, the necessary imports:: @@ -2041,12 +2038,12 @@ Finally, the Option subclass:: TYPE_CHECKER["complex"] = check_complex (If we didn't make a :func:`copy` of :attr:`Option.TYPE_CHECKER`, we would end -up modifying the :attr:`~Option.TYPE_CHECKER` attribute of :mod:`optparse`'s +up modifying the :attr:`~Option.TYPE_CHECKER` attribute of :mod:`!optparse`'s Option class. This being Python, nothing stops you from doing that except good manners and common sense.) That's it! Now you can write a script that uses the new option type just like -any other :mod:`optparse`\ -based script, except you have to instruct your +any other :mod:`!optparse`\ -based script, except you have to instruct your OptionParser to use MyOption instead of Option:: parser = OptionParser(option_class=MyOption) @@ -2066,10 +2063,10 @@ Adding new actions ^^^^^^^^^^^^^^^^^^ Adding new actions is a bit trickier, because you have to understand that -:mod:`optparse` has a couple of classifications for actions: +:mod:`!optparse` has a couple of classifications for actions: "store" actions - actions that result in :mod:`optparse` storing a value to an attribute of the + actions that result in :mod:`!optparse` storing a value to an attribute of the current OptionValues instance; these options require a :attr:`~Option.dest` attribute to be supplied to the Option constructor. @@ -2101,7 +2098,7 @@ of the following class attributes of Option (all are lists of strings): .. attribute:: Option.ALWAYS_TYPED_ACTIONS Actions that always take a type (i.e. whose options always take a value) are - additionally listed here. The only effect of this is that :mod:`optparse` + additionally listed here. The only effect of this is that :mod:`!optparse` assigns the default type, ``"string"``, to options with no explicit type whose action is listed in :attr:`ALWAYS_TYPED_ACTIONS`. @@ -2144,12 +2141,12 @@ Features of note: somewhere, so it goes in both :attr:`~Option.STORE_ACTIONS` and :attr:`~Option.TYPED_ACTIONS`. -* to ensure that :mod:`optparse` assigns the default type of ``"string"`` to +* to ensure that :mod:`!optparse` assigns the default type of ``"string"`` to ``"extend"`` actions, we put the ``"extend"`` action in :attr:`~Option.ALWAYS_TYPED_ACTIONS` as well. * :meth:`MyOption.take_action` implements just this one new action, and passes - control back to :meth:`Option.take_action` for the standard :mod:`optparse` + control back to :meth:`Option.take_action` for the standard :mod:`!optparse` actions. * ``values`` is an instance of the optparse_parser.Values class, which provides diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index bb9a7c86464..80818706173 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -36,7 +36,7 @@ the :mod:`glob` module.) Since different operating systems have different path name conventions, there are several versions of this module in the standard library. The - :mod:`os.path` module is always the path module suitable for the operating + :mod:`!os.path` module is always the path module suitable for the operating system Python is running on, and therefore usable for local paths. However, you can also import and use the individual modules if you want to manipulate a path that is *always* in one of the different formats. They all have the @@ -57,8 +57,9 @@ the :mod:`glob` module.) .. function:: abspath(path) Return a normalized absolutized version of the pathname *path*. On most - platforms, this is equivalent to calling the function :func:`normpath` as - follows: ``normpath(join(os.getcwd(), path))``. + platforms, this is equivalent to calling ``normpath(join(os.getcwd(), path))``. + + .. seealso:: :func:`os.path.join` and :func:`os.path.normpath`. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -96,15 +97,17 @@ the :mod:`glob` module.) .. function:: commonprefix(list, /) - Return the longest path prefix (taken character-by-character) that is a - prefix of all paths in *list*. If *list* is empty, return the empty string + Return the longest string prefix (taken character-by-character) that is a + prefix of all strings in *list*. If *list* is empty, return the empty string (``''``). - .. note:: + .. warning:: This function may return invalid paths because it works a - character at a time. To obtain a valid path, see - :func:`commonpath`. + character at a time. + If you need a **common path prefix**, then the algorithm + implemented in this function is not secure. Use + :func:`commonpath` for finding a common path prefix. :: @@ -117,6 +120,14 @@ the :mod:`glob` module.) .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. deprecated:: 3.15 + Deprecated in favor of :func:`os.path.commonpath` for path prefixes. + The :func:`os.path.commonprefix` function is being deprecated due to + having a misleading name and module. The function is not safe to use for + path prefixes despite being included in a module about path manipulation, + meaning it is easy to accidentally introduce path traversal + vulnerabilities into Python programs by using this function. + .. function:: dirname(path, /) @@ -243,6 +254,8 @@ the :mod:`glob` module.) begins with a slash, on Windows that it begins with two (back)slashes, or a drive letter, colon, and (back)slash together. + .. seealso:: :func:`abspath` + .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -357,14 +370,28 @@ the :mod:`glob` module.) concatenation of *path* and all members of *\*paths*, with exactly one directory separator following each non-empty part, except the last. That is, the result will only end in a separator if the last part is either empty or - ends in a separator. If a segment is an absolute path (which on Windows - requires both a drive and a root), then all previous segments are ignored and - joining continues from the absolute path segment. + ends in a separator. + + If a segment is an absolute path (which on Windows requires both a drive and + a root), then all previous segments are ignored and joining continues from the + absolute path segment. On Linux, for example:: + + >>> os.path.join('/home/foo', 'bar') + '/home/foo/bar' + >>> os.path.join('/home/foo', '/home/bar') + '/home/bar' On Windows, the drive is not reset when a rooted path segment (e.g., ``r'\foo'``) is encountered. If a segment is on a different drive or is an - absolute path, all previous segments are ignored and the drive is reset. Note - that since there is a current directory for each drive, + absolute path, all previous segments are ignored and the drive is reset. For + example:: + + >>> os.path.join('c:\\', 'foo') + 'c:\\foo' + >>> os.path.join('c:\\foo', 'd:\\bar') + 'd:\\bar' + + Note that since there is a current directory for each drive, ``os.path.join("c:", "foo")`` represents a path relative to the current directory on drive :file:`C:` (:file:`c:foo`), not :file:`c:\\foo`. @@ -449,7 +476,7 @@ the :mod:`glob` module.) .. versionchanged:: 3.10 The *strict* parameter was added. - .. versionchanged:: next + .. versionchanged:: 3.15 The :data:`ALL_BUT_LAST` and :data:`ALLOW_MISSING` values for the *strict* parameter was added. @@ -457,13 +484,13 @@ the :mod:`glob` module.) Special value used for the *strict* argument in :func:`realpath`. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: ALLOW_MISSING Special value used for the *strict* argument in :func:`realpath`. - .. versionadded:: next + .. versionadded:: 3.15 .. function:: relpath(path, start=os.curdir) @@ -527,8 +554,8 @@ the :mod:`glob` module.) *path* is empty, both *head* and *tail* are empty. Trailing slashes are stripped from *head* unless it is the root (one or more slashes only). In all cases, ``join(head, tail)`` returns a path to the same location as *path* - (but the strings may differ). Also see the functions :func:`dirname` and - :func:`basename`. + (but the strings may differ). Also see the functions :func:`join`, + :func:`dirname` and :func:`basename`. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 0333fe9f996..d2534b3e974 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -25,7 +25,7 @@ Notes on the availability of these functions: with the POSIX interface). * Extensions peculiar to a particular operating system are also available - through the :mod:`os` module, but using them is of course a threat to + through the :mod:`!os` module, but using them is of course a threat to portability. * All functions accepting path or file names accept both bytes and string @@ -34,7 +34,7 @@ Notes on the availability of these functions: * On VxWorks, os.popen, os.fork, os.execv and os.spawn*p* are not supported. -* On WebAssembly platforms, Android and iOS, large parts of the :mod:`os` module are +* On WebAssembly platforms, Android and iOS, large parts of the :mod:`!os` module are not available or behave differently. APIs related to processes (e.g. :func:`~os.fork`, :func:`~os.execve`) and resources (e.g. :func:`~os.nice`) are not available. Others like :func:`~os.getuid` and :func:`~os.getpid` are @@ -108,10 +108,10 @@ Python UTF-8 Mode .. versionadded:: 3.7 See :pep:`540` for more details. -.. versionchanged:: next +.. versionchanged:: 3.15 Python UTF-8 mode is now enabled by default (:pep:`686`). - It may be disabled with by setting :envvar:`PYTHONUTF8=0 ` as + It may be disabled by setting :envvar:`PYTHONUTF8=0 ` as an environment variable or by using the :option:`-X utf8=0 <-X>` command line option. The Python UTF-8 Mode ignores the :term:`locale encoding` and forces the usage @@ -185,7 +185,7 @@ process and user. of your home directory (on some platforms), and is equivalent to ``getenv("HOME")`` in C. - This mapping is captured the first time the :mod:`os` module is imported, + This mapping is captured the first time the :mod:`!os` module is imported, typically during Python startup as part of processing :file:`site.py`. Changes to the environment made after this time are not reflected in :data:`os.environ`, except for changes made by modifying :data:`os.environ` directly. @@ -216,8 +216,8 @@ process and user. You can delete items in this mapping to unset environment variables. :func:`unsetenv` will be called automatically when an item is deleted from - :data:`os.environ`, and when one of the :meth:`pop` or :meth:`clear` methods is - called. + :data:`os.environ`, and when one of the :meth:`~dict.pop` or + :meth:`~dict.clear` methods is called. .. seealso:: @@ -254,7 +254,7 @@ process and user. .. warning:: This function is not thread-safe. Calling it while the environment is - being modified in an other thread is an undefined behavior. Reading from + being modified in another thread is an undefined behavior. Reading from :data:`os.environ` or :data:`os.environb`, or calling :func:`os.getenv` while reloading, may return an empty result. @@ -430,8 +430,8 @@ process and user. associated with the effective user id of the process; the group access list may change over the lifetime of the process, it is not affected by calls to :func:`setgroups`, and its length is not limited to 16. The - deployment target value, :const:`MACOSX_DEPLOYMENT_TARGET`, can be - obtained with :func:`sysconfig.get_config_var`. + deployment target value can be obtained with + :func:`sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') `. .. function:: getlogin() @@ -558,7 +558,7 @@ process and user. .. function:: initgroups(username, gid, /) - Call the system initgroups() to initialize the group access list with all of + Call the system ``initgroups()`` to initialize the group access list with all of the groups of which the specified username is a member, plus the specified group id. @@ -1279,7 +1279,7 @@ as internal buffering of data. For a description of the flag and mode values, see the C run-time documentation; flag constants (like :const:`O_RDONLY` and :const:`O_WRONLY`) are defined in - the :mod:`os` module. In particular, on Windows adding + the :mod:`!os` module. In particular, on Windows adding :const:`O_BINARY` is needed to open files in binary mode. This function can support :ref:`paths relative to directory descriptors @@ -1294,8 +1294,8 @@ as internal buffering of data. This function is intended for low-level I/O. For normal usage, use the built-in function :func:`open`, which returns a :term:`file object` with - :meth:`~file.read` and :meth:`~file.write` methods (and many more). To - wrap a file descriptor in a file object, use :func:`fdopen`. + :meth:`~io.BufferedIOBase.read` and :meth:`~io.BufferedIOBase.write` methods. + To wrap a file descriptor in a file object, use :func:`fdopen`. .. versionchanged:: 3.3 Added the *dir_fd* parameter. @@ -1501,6 +1501,7 @@ or `the MSDN `_ on Windo - :data:`RWF_HIPRI` - :data:`RWF_NOWAIT` + - :data:`RWF_DONTCACHE` Return the total number of bytes actually read which can be less than the total capacity of all the objects. @@ -1546,6 +1547,24 @@ or `the MSDN `_ on Windo .. versionadded:: 3.7 +.. data:: RWF_DONTCACHE + + Use uncached buffered IO. + + .. availability:: Linux >= 6.14 + + .. versionadded:: 3.15 + + +.. data:: RWF_ATOMIC + + Write data atomically. Requires alignment to the device's atomic write unit. + + .. availability:: Linux >= 6.11 + + .. versionadded:: 3.15 + + .. function:: ptsname(fd, /) Return the name of the slave pseudo-terminal device associated with the @@ -1587,6 +1606,8 @@ or `the MSDN `_ on Windo - :data:`RWF_DSYNC` - :data:`RWF_SYNC` - :data:`RWF_APPEND` + - :data:`RWF_DONTCACHE` + - :data:`RWF_ATOMIC` Return the total number of bytes actually written. @@ -1649,7 +1670,7 @@ or `the MSDN `_ on Windo descriptor as returned by :func:`os.open` or :func:`pipe`. To read a "file object" returned by the built-in function :func:`open` or by :func:`popen` or :func:`fdopen`, or :data:`sys.stdin`, use its - :meth:`~file.read` or :meth:`~file.readline` methods. + :meth:`~io.TextIOBase.read` or :meth:`~io.IOBase.readline` methods. .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an @@ -1884,7 +1905,7 @@ or `the MSDN `_ on Windo descriptor as returned by :func:`os.open` or :func:`pipe`. To write a "file object" returned by the built-in function :func:`open` or by :func:`popen` or :func:`fdopen`, or :data:`sys.stdout` or :data:`sys.stderr`, use its - :meth:`~file.write` method. + :meth:`~io.TextIOBase.write` method. .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an @@ -1958,7 +1979,8 @@ can be inherited by child processes. Since Python 3.4, file descriptors created by Python are non-inheritable by default. On UNIX, non-inheritable file descriptors are closed in child processes at the -execution of a new program, other file descriptors are inherited. +execution of a new program, other file descriptors are inherited. Note that +non-inheritable file descriptors are still *inherited* by child processes on :func:`os.fork`. On Windows, non-inheritable handles and file descriptors are closed in child processes, except for standard streams (file descriptors 0, 1 and 2: stdin, stdout @@ -2002,12 +2024,12 @@ features: .. _path_fd: * **specifying a file descriptor:** - Normally the *path* argument provided to functions in the :mod:`os` module + Normally the *path* argument provided to functions in the :mod:`!os` module must be a string specifying a file path. However, some functions now alternatively accept an open file descriptor for their *path* argument. The function will then operate on the file referred to by the descriptor. - (For POSIX systems, Python will call the variant of the function prefixed - with ``f`` (e.g. call ``fchdir`` instead of ``chdir``).) + For POSIX systems, Python will call the variant of the function prefixed + with ``f`` (e.g. call ``fchdir`` instead of ``chdir``). You can check whether or not *path* can be specified as a file descriptor for a particular function on your platform using :data:`os.supports_fd`. @@ -2022,7 +2044,7 @@ features: * **paths relative to directory descriptors:** If *dir_fd* is not ``None``, it should be a file descriptor referring to a directory, and the path to operate on should be relative; path will then be relative to that directory. If the - path is absolute, *dir_fd* is ignored. (For POSIX systems, Python will call + path is absolute, *dir_fd* is ignored. For POSIX systems, Python will call the variant of the function with an ``at`` suffix and possibly prefixed with ``f`` (e.g. call ``faccessat`` instead of ``access``). @@ -2035,8 +2057,8 @@ features: * **not following symlinks:** If *follow_symlinks* is ``False``, and the last element of the path to operate on is a symbolic link, the function will operate on the symbolic link itself rather than the file - pointed to by the link. (For POSIX systems, Python will call the ``l...`` - variant of the function.) + pointed to by the link. For POSIX systems, Python will call the ``l...`` + variant of the function. You can check whether or not *follow_symlinks* is supported for a particular function on your platform using :data:`os.supports_follow_symlinks`. @@ -2387,6 +2409,10 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.15 + ``os.listdir(-1)`` now fails with ``OSError(errno.EBADF)`` rather than + listing the current directory. + .. function:: listdrives() @@ -2595,10 +2621,10 @@ features: Create a filesystem node (file, device special file or named pipe) named *path*. *mode* specifies both the permissions to use and the type of node - to be created, being combined (bitwise OR) with one of ``stat.S_IFREG``, - ``stat.S_IFCHR``, ``stat.S_IFBLK``, and ``stat.S_IFIFO`` (those constants are - available in :mod:`stat`). For ``stat.S_IFCHR`` and ``stat.S_IFBLK``, - *device* defines the newly created device special file (probably using + to be created, being combined (bitwise OR) with one of :const:`stat.S_IFREG`, + :const:`stat.S_IFCHR`, :const:`stat.S_IFBLK`, and :const:`stat.S_IFIFO`. + For :const:`stat.S_IFCHR` and :const:`stat.S_IFBLK`, *device* defines the + newly created device special file (probably using :func:`os.makedev`), otherwise it is ignored. This function can also support :ref:`paths relative to directory descriptors @@ -2616,13 +2642,13 @@ features: .. function:: major(device, /) Extract the device major number from a raw device number (usually the - :attr:`st_dev` or :attr:`st_rdev` field from :c:struct:`stat`). + :attr:`~stat_result.st_dev` or :attr:`~stat_result.st_rdev` field from :c:struct:`stat`). .. function:: minor(device, /) Extract the device minor number from a raw device number (usually the - :attr:`st_dev` or :attr:`st_rdev` field from :c:struct:`stat`). + :attr:`~stat_result.st_dev` or :attr:`~stat_result.st_rdev` field from :c:struct:`stat`). .. function:: makedev(major, minor, /) @@ -2630,6 +2656,13 @@ features: Compose a raw device number from the major and minor device numbers. +.. data:: NODEV + + Non-existent device. + + .. versionadded:: 3.15 + + .. function:: pathconf(path, name) Return system configuration information relevant to a named file. *name* @@ -2910,6 +2943,10 @@ features: .. versionchanged:: 3.7 Added support for :ref:`file descriptors ` on Unix. + .. versionchanged:: 3.15 + ``os.scandir(-1)`` now fails with ``OSError(errno.EBADF)`` rather than + listing the current directory. + .. class:: DirEntry @@ -3346,8 +3383,8 @@ features: .. versionchanged:: 3.8 On Windows, the :attr:`st_mode` member now identifies special - files as :const:`S_IFCHR`, :const:`S_IFIFO` or :const:`S_IFBLK` - as appropriate. + files as :const:`~stat.S_IFCHR`, :const:`~stat.S_IFIFO` or + :const:`~stat.S_IFBLK` as appropriate. .. versionchanged:: 3.12 On Windows, :attr:`st_ctime` is deprecated. Eventually, it will @@ -3365,6 +3402,359 @@ features: Added the :attr:`st_birthtime` member on Windows. +.. function:: statx(path, mask, *, flags=0, dir_fd=None, follow_symlinks=True) + + Get the status of a file or file descriptor by performing a :c:func:`!statx` + system call on the given path. + + *path* is a :term:`path-like object` or an open file descriptor. *mask* is a + combination of the module-level :const:`STATX_* ` constants + specifying the information to retrieve. *flags* is a combination of the + module-level :const:`AT_STATX_* ` constants and/or + :const:`AT_NO_AUTOMOUNT`. Returns a :class:`statx_result` object whose + :attr:`~os.statx_result.stx_mask` attribute specifies the information + actually retrieved (which may differ from *mask*). + + This function supports :ref:`specifying a file descriptor `, + :ref:`paths relative to directory descriptors `, and + :ref:`not following symlinks `. + + .. seealso:: The :manpage:`statx(2)` man page. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: 3.15 + + +.. class:: statx_result + + Information about a file returned by :func:`os.statx`. + + :class:`!statx_result` has the following attributes: + + .. attribute:: stx_atime + + Time of most recent access expressed in seconds. + + Equal to ``None`` if :data:`STATX_ATIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_atime_ns + + Time of most recent access expressed in nanoseconds as an integer. + + Equal to ``None`` if :data:`STATX_ATIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_atomic_write_segments_max + + Maximum iovecs for direct I/O with torn-write protection. + + Equal to ``None`` if :data:`STATX_WRITE_ATOMIC` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.11. + + .. attribute:: stx_atomic_write_unit_max + + Maximum size for direct I/O with torn-write protection. + + Equal to ``None`` if :data:`STATX_WRITE_ATOMIC` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.11. + + .. attribute:: stx_atomic_write_unit_max_opt + + Maximum optimized size for direct I/O with torn-write protection. + + Equal to ``None`` if :data:`STATX_WRITE_ATOMIC` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.16. + + .. attribute:: stx_atomic_write_unit_min + + Minimum size for direct I/O with torn-write protection. + + Equal to ``None`` if :data:`STATX_WRITE_ATOMIC` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.11. + + .. attribute:: stx_attributes + + Bitmask of :const:`STATX_ATTR_* ` constants + specifying the attributes of this file. + + .. attribute:: stx_attributes_mask + + A mask indicating which bits in :attr:`stx_attributes` are supported by + the VFS and the filesystem. + + .. attribute:: stx_blksize + + "Preferred" blocksize for efficient file system I/O. Writing to a file in + smaller chunks may cause an inefficient read-modify-rewrite. + + .. attribute:: stx_blocks + + Number of 512-byte blocks allocated for file. + This may be smaller than :attr:`stx_size`/512 when the file has holes. + + Equal to ``None`` if :data:`STATX_BLOCKS` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_btime + + Time of file creation expressed in seconds. + + Equal to ``None`` if :data:`STATX_BTIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_btime_ns + + Time of file creation expressed in nanoseconds as an integer. + + Equal to ``None`` if :data:`STATX_BTIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_ctime + + Time of most recent metadata change expressed in seconds. + + Equal to ``None`` if :data:`STATX_CTIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_ctime_ns + + Time of most recent metadata change expressed in nanoseconds as an + integer. + + Equal to ``None`` if :data:`STATX_CTIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_dev + + Identifier of the device on which this file resides. + + .. attribute:: stx_dev_major + + Major number of the device on which this file resides. + + .. attribute:: stx_dev_minor + + Minor number of the device on which this file resides. + + .. attribute:: stx_dio_mem_align + + Direct I/O memory buffer alignment requirement. + + Equal to ``None`` if :data:`STATX_DIOALIGN` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.1. + + .. attribute:: stx_dio_offset_align + + Direct I/O file offset alignment requirement. + + Equal to ``None`` if :data:`STATX_DIOALIGN` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.1. + + .. attribute:: stx_dio_read_offset_align + + Direct I/O file offset alignment requirement for reads. + + Equal to ``None`` if :data:`STATX_DIO_READ_ALIGN` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.14. + + .. attribute:: stx_gid + + Group identifier of the file owner. + + Equal to ``None`` if :data:`STATX_GID` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_ino + + Inode number. + + Equal to ``None`` if :data:`STATX_INO` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_mask + + Bitmask of :const:`STATX_* ` constants specifying the + information retrieved, which may differ from what was requested. + + .. attribute:: stx_mnt_id + + Mount identifier. + + Equal to ``None`` if :data:`STATX_MNT_ID` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 5.8. + + .. attribute:: stx_mode + + File mode: file type and file mode bits (permissions). + + Equal to ``None`` if :data:`STATX_TYPE | STATX_MODE ` + is missing from :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_mtime + + Time of most recent content modification expressed in seconds. + + Equal to ``None`` if :data:`STATX_MTIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_mtime_ns + + Time of most recent content modification expressed in nanoseconds as an + integer. + + Equal to ``None`` if :data:`STATX_MTIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_nlink + + Number of hard links. + + Equal to ``None`` if :data:`STATX_NLINK` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_rdev + + Type of device if an inode device. + + .. attribute:: stx_rdev_major + + Major number of the device this file represents. + + .. attribute:: stx_rdev_minor + + Minor number of the device this file represents. + + .. attribute:: stx_size + + Size of the file in bytes, if it is a regular file or a symbolic link. + The size of a symbolic link is the length of the pathname it contains, + without a terminating null byte. + + Equal to ``None`` if :data:`STATX_SIZE` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_subvol + + Subvolume identifier. + + Equal to ``None`` if :data:`STATX_SUBVOL` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.10. + + .. attribute:: stx_uid + + User identifier of the file owner. + + Equal to ``None`` if :data:`STATX_UID` is missing from + :attr:`~statx_result.stx_mask`. + + .. seealso:: The :manpage:`statx(2)` man page. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: 3.15 + + +.. data:: STATX_TYPE + STATX_MODE + STATX_NLINK + STATX_UID + STATX_GID + STATX_ATIME + STATX_MTIME + STATX_CTIME + STATX_INO + STATX_SIZE + STATX_BLOCKS + STATX_BASIC_STATS + STATX_BTIME + STATX_MNT_ID + STATX_DIOALIGN + STATX_MNT_ID_UNIQUE + STATX_SUBVOL + STATX_WRITE_ATOMIC + STATX_DIO_READ_ALIGN + + Bitflags for use in the *mask* parameter to :func:`os.statx`. Some of these + flags may be available even when their corresponding members in + :class:`statx_result` are not available. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: 3.15 + +.. data:: AT_STATX_FORCE_SYNC + + A flag for the :func:`os.statx` function. Requests that the kernel return + up-to-date information even when doing so is expensive (for example, + requiring a round trip to the server for a file on a network filesystem). + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: 3.15 + +.. data:: AT_STATX_DONT_SYNC + + A flag for the :func:`os.statx` function. Requests that the kernel return + cached information if possible. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: 3.15 + +.. data:: AT_STATX_SYNC_AS_STAT + + A flag for the :func:`os.statx` function. This flag is defined as ``0``, so + it has no effect, but it can be used to explicitly indicate neither + :data:`AT_STATX_FORCE_SYNC` nor :data:`AT_STATX_DONT_SYNC` is being passed. + In the absence of the other two flags, the kernel will generally return + information as fresh as :func:`os.stat` would return. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: 3.15 + + +.. data:: AT_NO_AUTOMOUNT + + If the final component of a path is an automount point, operate on the + automount point instead of performing the automount. On Linux, + :func:`os.stat`, :func:`os.fstat` and :func:`os.lstat` always behave this + way. + + .. availability:: Linux. + + .. versionadded:: 3.15 + + .. function:: statvfs(path) Perform a :c:func:`!statvfs` system call on the given path. The return value is @@ -3413,7 +3803,7 @@ features: .. data:: supports_dir_fd - A :class:`set` object indicating which functions in the :mod:`os` + A :class:`set` object indicating which functions in the :mod:`!os` module accept an open file descriptor for their *dir_fd* parameter. Different platforms provide different features, and the underlying functionality Python uses to implement the *dir_fd* parameter is not @@ -3458,7 +3848,7 @@ features: .. data:: supports_fd A :class:`set` object indicating which functions in the - :mod:`os` module permit specifying their *path* parameter as an open file + :mod:`!os` module permit specifying their *path* parameter as an open file descriptor on the local platform. Different platforms provide different features, and the underlying functionality Python uses to accept open file descriptors as *path* arguments is not available on all platforms Python @@ -3477,7 +3867,7 @@ features: .. data:: supports_follow_symlinks - A :class:`set` object indicating which functions in the :mod:`os` module + A :class:`set` object indicating which functions in the :mod:`!os` module accept ``False`` for their *follow_symlinks* parameter on the local platform. Different platforms provide different features, and the underlying functionality Python uses to implement *follow_symlinks* is not available @@ -3502,6 +3892,9 @@ features: Create a symbolic link pointing to *src* named *dst*. + The *src* parameter refers to the target of the link (the file or directory being linked to), + and *dst* is the name of the link being created. + On Windows, a symlink represents either a file or a directory, and does not morph to the target dynamically. If the target is present, the type of the symlink will be created to match. Otherwise, the symlink will be created @@ -3600,7 +3993,8 @@ features: where each member is an int expressing nanoseconds. - If *times* is not ``None``, it must be a 2-tuple of the form ``(atime, mtime)`` - where each member is an int or float expressing seconds. + where each member is a real number expressing seconds, + rounded down to nanoseconds. - If *times* is ``None`` and *ns* is unspecified, this is equivalent to specifying ``ns=(atime_ns, mtime_ns)`` where both times are the current time. @@ -3627,6 +4021,9 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.15 + Accepts any real numbers as *times*, not only integers or floats. + .. function:: walk(top, topdown=True, onerror=None, followlinks=False) @@ -3873,7 +4270,7 @@ features: import os # semaphore with start value '1' - fd = os.eventfd(1, os.EFD_SEMAPHORE | os.EFC_CLOEXEC) + fd = os.eventfd(1, os.EFD_SEMAPHORE | os.EFD_CLOEXEC) try: # acquire semaphore v = os.eventfd_read(fd) @@ -3982,7 +4379,7 @@ Naturally, they are all only available on Linux. except it includes any time that the system is suspended. The file descriptor's behaviour can be modified by specifying a *flags* value. - Any of the following variables may used, combined using bitwise OR + Any of the following variables may be used, combined using bitwise OR (the ``|`` operator): - :const:`TFD_NONBLOCK` @@ -4014,7 +4411,7 @@ Naturally, they are all only available on Linux. *fd* must be a valid timer file descriptor. The timer's behaviour can be modified by specifying a *flags* value. - Any of the following variables may used, combined using bitwise OR + Any of the following variables may be used, combined using bitwise OR (the ``|`` operator): - :const:`TFD_TIMER_ABSTIME` @@ -4032,7 +4429,7 @@ Naturally, they are all only available on Linux. the timer will fire when the timer's clock (set by *clockid* in :func:`timerfd_create`) reaches *initial* seconds. - The timer's interval is set by the *interval* :py:class:`float`. + The timer's interval is set by the *interval* real number. If *interval* is zero, the timer only fires once, on the initial expiration. If *interval* is greater than zero, the timer fires every time *interval* seconds have elapsed since the previous expiration. @@ -4083,7 +4480,7 @@ Naturally, they are all only available on Linux. Return a two-item tuple of floats (``next_expiration``, ``interval``). - ``next_expiration`` denotes the relative time until next the timer next fires, + ``next_expiration`` denotes the relative time until the timer next fires, regardless of if the :const:`TFD_TIMER_ABSTIME` flag is set. ``interval`` denotes the timer's interval. @@ -4185,6 +4582,10 @@ These functions are all available on Linux only. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.15 + ``os.listxattr(-1)`` now fails with ``OSError(errno.EBADF)`` rather than + listing extended attributes of the current directory. + .. function:: removexattr(path, attribute, *, follow_symlinks=True) @@ -4263,10 +4664,10 @@ to be ignored. .. function:: abort() - Generate a :const:`SIGABRT` signal to the current process. On Unix, the default + Generate a :const:`~signal.SIGABRT` signal to the current process. On Unix, the default behavior is to produce a core dump; on Windows, the process immediately returns an exit code of ``3``. Be aware that calling this function will not call the - Python signal handler registered for :const:`SIGABRT` with + Python signal handler registered for :const:`~signal.SIGABRT` with :func:`signal.signal`. @@ -4319,7 +4720,7 @@ to be ignored. The current process is replaced immediately. Open file objects and descriptors are not flushed, so if there may be data buffered on these open files, you should flush them using - :func:`sys.stdout.flush` or :func:`os.fsync` before calling an + :func:`~io.IOBase.flush` or :func:`os.fsync` before calling an :func:`exec\* ` function. The "l" and "v" variants of the :func:`exec\* ` functions differ in how @@ -4570,6 +4971,8 @@ written in Python, such as a mail server's external command delivery program. master end of the pseudo-terminal. For a more portable approach, use the :mod:`pty` module. If an error occurs :exc:`OSError` is raised. + The returned file descriptor *fd* is :ref:`non-inheritable `. + .. audit-event:: os.forkpty "" os.forkpty .. warning:: @@ -4586,6 +4989,9 @@ written in Python, such as a mail server's external command delivery program. threads, this now raises a :exc:`DeprecationWarning`. See the longer explanation on :func:`os.fork`. + .. versionchanged:: 3.15 + The returned file descriptor is now made non-inheritable. + .. availability:: Unix, not WASI, not Android, not iOS. @@ -4704,9 +5110,8 @@ written in Python, such as a mail server's external command delivery program. Use :class:`subprocess.Popen` or :func:`subprocess.run` to control options like encodings. - .. deprecated:: 3.14 - The function is :term:`soft deprecated` and should no longer be used to - write new code. The :mod:`subprocess` module is recommended instead. + .. soft-deprecated:: 3.14 + The :mod:`subprocess` module is recommended instead. .. function:: posix_spawn(path, argv, env, *, file_actions=None, \ @@ -4934,9 +5339,8 @@ written in Python, such as a mail server's external command delivery program. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. deprecated:: 3.14 - These functions are :term:`soft deprecated` and should no longer be used - to write new code. The :mod:`subprocess` module is recommended instead. + .. soft-deprecated:: 3.14 + The :mod:`subprocess` module is recommended instead. .. data:: P_NOWAIT @@ -5610,7 +6014,7 @@ Miscellaneous System Information .. versionchanged:: 3.13 If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, - :func:`cpu_count` returns the overridden value *n*. + :func:`cpu_count` returns the override value *n*. .. function:: getloadavg() @@ -5632,7 +6036,7 @@ Miscellaneous System Information in the **system**. If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, - :func:`process_cpu_count` returns the overridden value *n*. + :func:`process_cpu_count` returns the override value *n*. See also the :func:`sched_getaffinity` function. diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 79e0b7f09ea..2867015042e 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -486,6 +486,10 @@ Pure paths provide the following methods and properties: >>> PurePosixPath('my/library').stem 'library' + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. + .. method:: PurePath.as_posix() @@ -1331,6 +1335,10 @@ Reading directories PosixPath('setup.py'), PosixPath('test_pathlib.py')] + .. note:: + The paths are returned in no particular order. + If you need a specific order, sort the results. + .. seealso:: :ref:`pathlib-pattern-language` documentation. @@ -1343,6 +1351,11 @@ Reading directories ``False``, this method follows symlinks except when expanding "``**``" wildcards. Set *recurse_symlinks* to ``True`` to always follow symlinks. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob .. versionchanged:: 3.12 @@ -1365,6 +1378,15 @@ Reading directories Glob the given relative *pattern* recursively. This is like calling :func:`Path.glob` with "``**/``" added in front of the *pattern*. + .. note:: + The paths are returned in no particular order. + If you need a specific order, sort the results. + + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. seealso:: :ref:`pathlib-pattern-language` and :meth:`Path.glob` documentation. @@ -1875,7 +1897,7 @@ Below is a table mapping various :mod:`os` functions to their corresponding :class:`PurePath`/:class:`Path` equivalent. ===================================== ============================================== -:mod:`os` and :mod:`os.path` :mod:`pathlib` +:mod:`os` and :mod:`os.path` :mod:`!pathlib` ===================================== ============================================== :func:`os.path.dirname` :attr:`PurePath.parent` :func:`os.path.basename` :attr:`PurePath.name` @@ -1934,7 +1956,7 @@ Protocols :synopsis: pathlib types for static type checking -The :mod:`pathlib.types` module provides types for static type checking. +The :mod:`!pathlib.types` module provides types for static type checking. .. versionadded:: 3.14 diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 90dc6648045..bfe017a5c8f 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -1,7 +1,7 @@ .. _debugger: -:mod:`pdb` --- The Python Debugger -================================== +:mod:`!pdb` --- The Python Debugger +=================================== .. module:: pdb :synopsis: The Python debugger for interactive interpreters. @@ -12,7 +12,7 @@ -------------- -The module :mod:`pdb` defines an interactive source code debugger for Python +The module :mod:`!pdb` defines an interactive source code debugger for Python programs. It supports setting (conditional) breakpoints and single stepping at the source line level, inspection of stack frames, source code listing, and evaluation of arbitrary Python code in the context of any stack frame. It also @@ -76,9 +76,13 @@ The debugger's prompt is ``(Pdb)``, which is the indicator that you are in debug .. _pdb-cli: + +Command-line interface +---------------------- + .. program:: pdb -You can also invoke :mod:`pdb` from the command line to debug other scripts. For +You can also invoke :mod:`!pdb` from the command line to debug other scripts. For example:: python -m pdb [-c command] (-m module | -p pid | pyfile) [args ...] @@ -334,7 +338,7 @@ access further features, you have to do this yourself: .. _debugger-commands: -Debugger Commands +Debugger commands ----------------- The commands recognized by the debugger are listed below. Most commands can be @@ -516,7 +520,8 @@ can be overridden by the local file. To remove all commands from a breakpoint, type ``commands`` and follow it immediately with ``end``; that is, give no commands. - With no *bpnumber* argument, ``commands`` refers to the last breakpoint set. + With no *bpnumber* argument, ``commands`` refers to the most recently set + breakpoint that still exists. You can use breakpoint commands to start your program up again. Simply use the :pdbcmd:`continue` command, or :pdbcmd:`step`, diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 3a9b66ec7e7..f8975c2f428 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -4,9 +4,6 @@ .. module:: pickle :synopsis: Convert Python objects to streams of bytes and back. -.. sectionauthor:: Jim Kerr . -.. sectionauthor:: Barry Warsaw - **Source code:** :source:`Lib/pickle.py` .. index:: @@ -19,7 +16,7 @@ -------------- -The :mod:`pickle` module implements binary protocols for serializing and +The :mod:`!pickle` module implements binary protocols for serializing and de-serializing a Python object structure. *"Pickling"* is the process whereby a Python object hierarchy is converted into a byte stream, and *"unpickling"* is the inverse operation, whereby a byte stream @@ -50,27 +47,14 @@ Comparison with ``marshal`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Python has a more primitive serialization module called :mod:`marshal`, but in -general :mod:`pickle` should always be the preferred way to serialize Python +general :mod:`!pickle` should always be the preferred way to serialize Python objects. :mod:`marshal` exists primarily to support Python's :file:`.pyc` files. -The :mod:`pickle` module differs from :mod:`marshal` in several significant ways: - -* The :mod:`pickle` module keeps track of the objects it has already serialized, - so that later references to the same object won't be serialized again. - :mod:`marshal` doesn't do this. - - This has implications both for recursive objects and object sharing. Recursive - objects are objects that contain references to themselves. These are not - handled by marshal, and in fact, attempting to marshal recursive objects will - crash your Python interpreter. Object sharing happens when there are multiple - references to the same object in different places in the object hierarchy being - serialized. :mod:`pickle` stores such objects only once, and ensures that all - other references point to the master copy. Shared objects remain shared, which - can be very important for mutable objects. +The :mod:`!pickle` module differs from :mod:`marshal` in several significant ways: * :mod:`marshal` cannot be used to serialize user-defined classes and their - instances. :mod:`pickle` can save and restore class instances transparently, + instances. :mod:`!pickle` can save and restore class instances transparently, however the class definition must be importable and live in the same module as when the object was stored. @@ -78,7 +62,7 @@ The :mod:`pickle` module differs from :mod:`marshal` in several significant ways across Python versions. Because its primary job in life is to support :file:`.pyc` files, the Python implementers reserve the right to change the serialization format in non-backwards compatible ways should the need arise. - The :mod:`pickle` serialization format is guaranteed to be backwards compatible + The :mod:`!pickle` serialization format is guaranteed to be backwards compatible across Python releases provided a compatible pickle protocol is chosen and pickling and unpickling code deals with Python 2 to Python 3 type differences if your data is crossing that unique breaking change language boundary. @@ -123,17 +107,17 @@ Data stream format .. index:: single: External Data Representation -The data format used by :mod:`pickle` is Python-specific. This has the +The data format used by :mod:`!pickle` is Python-specific. This has the advantage that there are no restrictions imposed by external standards such as JSON (which can't represent pointer sharing); however it means that non-Python programs may not be able to reconstruct pickled Python objects. -By default, the :mod:`pickle` data format uses a relatively compact binary +By default, the :mod:`!pickle` data format uses a relatively compact binary representation. If you need optimal size characteristics, you can efficiently :doc:`compress ` pickled data. The module :mod:`pickletools` contains tools for analyzing data streams -generated by :mod:`pickle`. :mod:`pickletools` source code has extensive +generated by :mod:`!pickle`. :mod:`pickletools` source code has extensive comments about opcodes used by pickle protocols. There are currently 6 different protocols which can be used for pickling. @@ -167,9 +151,9 @@ to read the pickle produced. .. note:: Serialization is a more primitive notion than persistence; although - :mod:`pickle` reads and writes file objects, it does not handle the issue of + :mod:`!pickle` reads and writes file objects, it does not handle the issue of naming persistent objects, nor the (even more complicated) issue of concurrent - access to persistent objects. The :mod:`pickle` module can transform a complex + access to persistent objects. The :mod:`!pickle` module can transform a complex object into a byte stream and it can transform the byte stream into an object with the same internal structure. Perhaps the most obvious thing to do with these byte streams is to write them onto a file, but it is also conceivable to @@ -186,7 +170,7 @@ Similarly, to de-serialize a data stream, you call the :func:`loads` function. However, if you want more control over serialization and de-serialization, you can create a :class:`Pickler` or an :class:`Unpickler` object, respectively. -The :mod:`pickle` module provides the following constants: +The :mod:`!pickle` module provides the following constants: .. data:: HIGHEST_PROTOCOL @@ -217,7 +201,7 @@ The :mod:`pickle` module provides the following constants: The default protocol is 5. -The :mod:`pickle` module provides the following functions to make the pickling +The :mod:`!pickle` module provides the following functions to make the pickling process more convenient: .. function:: dump(obj, file, protocol=None, *, fix_imports=True, buffer_callback=None) @@ -275,7 +259,7 @@ process more convenient: The *buffers* argument was added. -The :mod:`pickle` module defines three exceptions: +The :mod:`!pickle` module defines three exceptions: .. exception:: PickleError @@ -300,7 +284,7 @@ The :mod:`pickle` module defines three exceptions: IndexError. -The :mod:`pickle` module exports three classes, :class:`Pickler`, +The :mod:`!pickle` module exports three classes, :class:`Pickler`, :class:`Unpickler` and :class:`PickleBuffer`: .. class:: Pickler(file, protocol=None, *, fix_imports=True, buffer_callback=None) @@ -532,7 +516,7 @@ The following types can be pickled: * classes accessible from the top level of a module; -* instances of such classes whose the result of calling :meth:`~object.__getstate__` +* instances of such classes for which the result of calling :meth:`~object.__getstate__` is picklable (see section :ref:`pickle-inst` for details). Attempts to pickle unpicklable objects will raise the :exc:`PicklingError` @@ -601,7 +585,7 @@ methods: .. method:: object.__getnewargs_ex__() - In protocols 2 and newer, classes that implements the + In protocols 2 and newer, classes that implement the :meth:`__getnewargs_ex__` method can dictate the values passed to the :meth:`__new__` method upon unpickling. The method must return a pair ``(args, kwargs)`` where *args* is a tuple of positional arguments @@ -773,13 +757,13 @@ Persistence of External Objects single: persistent_id (pickle protocol) single: persistent_load (pickle protocol) -For the benefit of object persistence, the :mod:`pickle` module supports the +For the benefit of object persistence, the :mod:`!pickle` module supports the notion of a reference to an object outside the pickled data stream. Such objects are referenced by a persistent ID, which should be either a string of alphanumeric characters (for protocol 0) [#]_ or just an arbitrary object (for any newer protocol). -The resolution of such persistent IDs is not defined by the :mod:`pickle` +The resolution of such persistent IDs is not defined by the :mod:`!pickle` module; it will delegate this resolution to the user-defined methods on the pickler and unpickler, :meth:`~Pickler.persistent_id` and :meth:`~Unpickler.persistent_load` respectively. @@ -973,10 +957,10 @@ Out-of-band Buffers .. versionadded:: 3.8 -In some contexts, the :mod:`pickle` module is used to transfer massive amounts +In some contexts, the :mod:`!pickle` module is used to transfer massive amounts of data. Therefore, it can be important to minimize the number of memory copies, to preserve performance and resource consumption. However, normal -operation of the :mod:`pickle` module, as it transforms a graph-like structure +operation of the :mod:`!pickle` module, as it transforms a graph-like structure of objects into a sequential stream of bytes, intrinsically involves copying data to and from the pickle stream. @@ -995,8 +979,8 @@ for any large data. A :class:`PickleBuffer` object *signals* that the underlying buffer is eligible for out-of-band data transfer. Those objects remain compatible -with normal usage of the :mod:`pickle` module. However, consumers can also -opt-in to tell :mod:`pickle` that they will handle those buffers by +with normal usage of the :mod:`!pickle` module. However, consumers can also +opt-in to tell :mod:`!pickle` that they will handle those buffers by themselves. Consumer API @@ -1172,7 +1156,7 @@ Performance Recent versions of the pickle protocol (from protocol 2 and upwards) feature efficient binary encodings for several common features and built-in types. -Also, the :mod:`pickle` module has a transparent optimizer written in C. +Also, the :mod:`!pickle` module has a transparent optimizer written in C. .. _pickle-example: @@ -1215,7 +1199,7 @@ The following example reads the resulting pickled data. :: Command-line interface ---------------------- -The :mod:`pickle` module can be invoked as a script from the command line, +The :mod:`!pickle` module can be invoked as a script from the command line, it will display contents of the pickle files. However, when the pickle file that you want to examine comes from an untrusted source, ``-m pickletools`` is a safer option because it does not execute pickle bytecode, see @@ -1243,7 +1227,7 @@ The following option is accepted: Tools for working with and analyzing pickled data. Module :mod:`shelve` - Indexed databases of objects; uses :mod:`pickle`. + Indexed databases of objects; uses :mod:`!pickle`. Module :mod:`copy` Shallow and deep object copying. diff --git a/Doc/library/pickletools.rst b/Doc/library/pickletools.rst index 30fc2962e0b..e753ad3b08b 100644 --- a/Doc/library/pickletools.rst +++ b/Doc/library/pickletools.rst @@ -15,7 +15,7 @@ This module contains various constants relating to the intimate details of the few useful functions for analyzing pickled data. The contents of this module are useful for Python core developers who are working on the :mod:`pickle`; ordinary users of the :mod:`pickle` module probably won't find the -:mod:`pickletools` module relevant. +:mod:`!pickletools` module relevant. .. _pickletools-cli: @@ -79,6 +79,9 @@ Command-line options A pickle file to read, or ``-`` to indicate reading from standard input. +.. versionadded:: next + Output is in color by default and can be + :ref:`controlled using environment variables `. Programmatic interface diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 47d24b6f7d0..aa7dd71c132 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -151,24 +151,48 @@ support. :meth:`get_data ` API. The *package* argument should be the name of a package, in standard module format (``foo.bar``). The *resource* argument should be in the form of a relative - filename, using ``/`` as the path separator. The parent directory name - ``..`` is not allowed, and nor is a rooted name (starting with a ``/``). + filename, using ``/`` as the path separator. The function returns a binary string that is the contents of the specified resource. + This function uses the :term:`loader` method + :func:`~importlib.abc.FileLoader.get_data` + to support modules installed in the filesystem, but also in zip files, + databases, or elsewhere. + For packages located in the filesystem, which have already been imported, this is the rough equivalent of:: d = os.path.dirname(sys.modules[package].__file__) data = open(os.path.join(d, resource), 'rb').read() + Like the :func:`open` function, :func:`!get_data` can follow parent + directories (``../``) and absolute paths (starting with ``/`` or ``C:/``, + for example). + It can open compilation/installation artifacts like ``.py`` and ``.pyc`` + files or files with :func:`reserved filenames `. + To be compatible with non-filesystem loaders, avoid using these features. + + .. warning:: + + This function is intended for trusted input. + It does not verify that *resource* "belongs" to *package*. + + If you use a user-provided *resource* path, consider verifying it. + For example, require an alphanumeric filename with a known extension, or + install and check a list of known resources. + If the package cannot be located or loaded, or it uses a :term:`loader` which does not support :meth:`get_data `, then ``None`` is returned. In particular, the :term:`loader` for :term:`namespace packages ` does not support :meth:`get_data `. + .. seealso:: + + The :mod:`importlib.resources` module provides structured access to + module resources. .. function:: resolve_name(name) diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 88affb5eea2..1d30966794f 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -4,9 +4,6 @@ .. module:: platform :synopsis: Retrieves as much platform identifying data as possible. -.. moduleauthor:: Marc-André Lemburg -.. sectionauthor:: Bjorn Pettersen - **Source code:** :source:`Lib/platform.py` -------------- @@ -56,6 +53,8 @@ Cross platform Returns the machine type, e.g. ``'AMD64'``. An empty string is returned if the value cannot be determined. + The output is platform-dependent and may differ in casing and naming conventions. + .. function:: node() @@ -355,7 +354,7 @@ Android platform Command-line usage ------------------ -:mod:`platform` can also be invoked directly using the :option:`-m` +:mod:`!platform` can also be invoked directly using the :option:`-m` switch of the interpreter:: python -m platform [--terse] [--nonaliased] [{nonaliased,terse} ...] diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 415c4b45c4f..72140e41675 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -4,10 +4,6 @@ .. module:: plistlib :synopsis: Generate and parse Apple plist files. -.. moduleauthor:: Jack Jansen -.. sectionauthor:: Georg Brandl -.. (harvested from docstrings in the original file) - **Source code:** :source:`Lib/plistlib.py` .. index:: @@ -22,7 +18,7 @@ and XML plist files. The property list (``.plist``) file format is a simple serialization supporting basic object types, like dictionaries, lists, numbers and strings. Usually the -top level object is a dictionary. +top level object is a dictionary or a frozen dictionary. To write out and to parse a plist file, use the :func:`dump` and :func:`load` functions. @@ -184,7 +180,7 @@ Examples Generating a plist:: - import datetime + import datetime as dt import plistlib pl = dict( @@ -200,7 +196,7 @@ Generating a plist:: ), someData = b"", someMoreData = b"" * 10, - aDate = datetime.datetime.now() + aDate = dt.datetime.now() ) print(plistlib.dumps(pl).decode()) diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst index 23f20b00e6d..cd3a58016e9 100644 --- a/Doc/library/poplib.rst +++ b/Doc/library/poplib.rst @@ -4,9 +4,6 @@ .. module:: poplib :synopsis: POP3 protocol client (requires sockets). -.. sectionauthor:: Andrew T. Csillag -.. revised by ESR, January 2000 - **Source code:** :source:`Lib/poplib.py` .. index:: pair: POP3; protocol @@ -30,7 +27,7 @@ mailserver supports IMAP, you would be better off using the .. include:: ../includes/wasm-notavail.rst -The :mod:`poplib` module provides two classes: +The :mod:`!poplib` module provides two classes: .. class:: POP3(host, port=POP3_PORT[, timeout]) @@ -86,7 +83,7 @@ The :mod:`poplib` module provides two classes: .. versionchanged:: 3.12 The deprecated *keyfile* and *certfile* parameters have been removed. -One exception is defined as an attribute of the :mod:`poplib` module: +One exception is defined as an attribute of the :mod:`!poplib` module: .. exception:: error_proto diff --git a/Doc/library/posix.rst b/Doc/library/posix.rst index 14ab3e91e8a..7d43b5d4cf7 100644 --- a/Doc/library/posix.rst +++ b/Doc/library/posix.rst @@ -2,7 +2,6 @@ ==================================================== .. module:: posix - :platform: Unix :synopsis: The most common POSIX system calls (normally used via module os). -------------- @@ -17,10 +16,10 @@ interface). **Do not import this module directly.** Instead, import the module :mod:`os`, which provides a *portable* version of this interface. On Unix, the :mod:`os` -module provides a superset of the :mod:`posix` interface. On non-Unix operating -systems the :mod:`posix` module is not available, but a subset is always +module provides a superset of the :mod:`!posix` interface. On non-Unix operating +systems the :mod:`!posix` module is not available, but a subset is always available through the :mod:`os` interface. Once :mod:`os` is imported, there is -*no* performance penalty in using it instead of :mod:`posix`. In addition, +*no* performance penalty in using it instead of :mod:`!posix`. In addition, :mod:`os` provides some additional functionality, such as automatically calling :func:`~os.putenv` when an entry in ``os.environ`` is changed. @@ -37,8 +36,6 @@ Large File Support single: large files single: file; large files -.. sectionauthor:: Steve Clift - Several operating systems (including AIX and Solaris) provide support for files that are larger than 2 GiB from a C programming model where :c:expr:`int` and :c:expr:`long` are 32-bit values. This is typically accomplished @@ -67,7 +64,7 @@ Notable Module Contents ----------------------- In addition to many functions described in the :mod:`os` module documentation, -:mod:`posix` defines the following data item: +:mod:`!posix` defines the following data item: .. data:: environ @@ -91,4 +88,4 @@ In addition to many functions described in the :mod:`os` module documentation, which updates the environment on modification. Note also that updating :data:`os.environ` will render this dictionary obsolete. Use of the :mod:`os` module version of this is recommended over direct access to the - :mod:`posix` module. + :mod:`!posix` module. diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index 2985f31bacb..4f043fbb3a4 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -4,14 +4,11 @@ .. module:: pprint :synopsis: Data pretty printer. -.. moduleauthor:: Fred L. Drake, Jr. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/pprint.py` -------------- -The :mod:`pprint` module provides a capability to "pretty-print" arbitrary +The :mod:`!pprint` module provides a capability to "pretty-print" arbitrary Python data structures in a form which can be used as input to the interpreter. If the formatted structures include objects which are not fundamental Python types, the representation may not be loadable. This may be the case if objects @@ -22,8 +19,6 @@ The formatted representation keeps objects on a single line if it can, and breaks them onto multiple lines if they don't fit within the allowed width, adjustable by the *width* parameter defaulting to 80 characters. -Dictionaries are sorted by key before the display is computed. - .. versionchanged:: 3.9 Added support for pretty-printing :class:`types.SimpleNamespace`. @@ -36,7 +31,8 @@ Functions --------- .. function:: pp(object, stream=None, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=False, underscore_numbers=False) + compact=False, expand=False, sort_dicts=False, \ + underscore_numbers=False) Prints the formatted representation of *object*, followed by a newline. This function may be used in the interactive interpreter @@ -74,6 +70,13 @@ Functions each item of a sequence will be formatted on a separate line, otherwise as many items as will fit within the *width* will be formatted on each output line. + Incompatible with *expand*. + + :param bool expand: + If ``True``, + opening parentheses and brackets will be followed by a newline and the + following content will be indented by one level, similar to + pretty-printed JSON. Incompatible with *compact*. :param bool sort_dicts: If ``True``, dictionaries will be formatted with @@ -100,7 +103,8 @@ Functions .. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) + compact=False, expand=False, sort_dicts=True, \ + underscore_numbers=False) Alias for :func:`~pprint.pp` with *sort_dicts* set to ``True`` by default, which would automatically sort the dictionaries' keys, @@ -108,10 +112,11 @@ Functions .. function:: pformat(object, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) + compact=False, expand=False, sort_dicts=True, \ + underscore_numbers=False) Return the formatted representation of *object* as a string. *indent*, - *width*, *depth*, *compact*, *sort_dicts* and *underscore_numbers* are + *width*, *depth*, *compact*, *expand*, *sort_dicts* and *underscore_numbers* are passed to the :class:`PrettyPrinter` constructor as formatting parameters and their meanings are as described in the documentation above. @@ -155,7 +160,8 @@ PrettyPrinter Objects .. index:: single: ...; placeholder .. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) + compact=False, expand=False, sort_dicts=True, \ + underscore_numbers=False) Construct a :class:`PrettyPrinter` instance. @@ -179,6 +185,22 @@ PrettyPrinter Objects 'knights', 'ni'], 'spam', 'eggs', 'lumberjack', 'knights', 'ni'] + >>> pp = pprint.PrettyPrinter(width=41, expand=True, indent=3) + >>> pp.pprint(stuff) + [ + [ + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni', + ], + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni', + ] >>> tup = ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', ... ('parrot', ('fresh fruit',)))))))) >>> pp = pprint.PrettyPrinter(depth=6) @@ -198,6 +220,9 @@ PrettyPrinter Objects .. versionchanged:: 3.11 No longer attempts to write to :data:`!sys.stdout` if it is ``None``. + .. versionchanged:: 3.15 + Added the *expand* parameter. + :class:`PrettyPrinter` instances have the following methods: @@ -420,3 +445,72 @@ cannot be split, the specified width will be exceeded:: 'requires_python': None, 'summary': 'A sample Python project', 'version': '1.2.0'} + +Lastly, we can format like pretty-printed JSON with the *expand* parameter. +Best results are achieved with a higher *indent* value:: + + >>> pprint.pp(project_info, indent=4, expand=True) + { + 'author': 'The Python Packaging Authority', + 'author_email': 'pypa-dev@googlegroups.com', + 'bugtrack_url': None, + 'classifiers': [ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Topic :: Software Development :: Build Tools', + ], + 'description': 'A sample Python project\n' + '=======================\n' + '\n' + 'This is the description file for the project.\n' + '\n' + 'The file should use UTF-8 encoding and be written using ReStructured ' + 'Text. It\n' + 'will be used to generate the project webpage on PyPI, and should be ' + 'written for\n' + 'that purpose.\n' + '\n' + 'Typical contents for this file would include an overview of the project, ' + 'basic\n' + 'usage examples, etc. Generally, including the project changelog in here ' + 'is not\n' + 'a good idea, although a simple "What\'s New" section for the most recent ' + 'version\n' + 'may be appropriate.', + 'description_content_type': None, + 'docs_url': None, + 'download_url': 'UNKNOWN', + 'downloads': {'last_day': -1, 'last_month': -1, 'last_week': -1}, + 'dynamic': None, + 'home_page': 'https://github.com/pypa/sampleproject', + 'keywords': 'sample setuptools development', + 'license': 'MIT', + 'license_expression': None, + 'license_files': None, + 'maintainer': None, + 'maintainer_email': None, + 'name': 'sampleproject', + 'package_url': 'https://pypi.org/project/sampleproject/', + 'platform': 'UNKNOWN', + 'project_url': 'https://pypi.org/project/sampleproject/', + 'project_urls': { + 'Download': 'UNKNOWN', + 'Homepage': 'https://github.com/pypa/sampleproject', + }, + 'provides_extra': None, + 'release_url': 'https://pypi.org/project/sampleproject/1.2.0/', + 'requires_dist': None, + 'requires_python': None, + 'summary': 'A sample Python project', + 'version': '1.2.0', + 'yanked': False, + 'yanked_reason': None, + } diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index faf8079db3d..f7e85d15987 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -1,523 +1,64 @@ .. _profile: -******************** -The Python Profilers -******************** +**************************************** +:mod:`!profile` --- Pure Python profiler +**************************************** -**Source code:** :source:`Lib/profile.py`, :source:`Lib/pstats.py`, and :source:`Lib/profile/sample.py` +.. module:: profile + :synopsis: Pure Python profiler (deprecated). + :deprecated: + +**Source code:** :source:`Lib/profile.py` -------------- -.. _profiler-introduction: +.. deprecated-removed:: 3.15 3.17 -Introduction to the profilers -============================= +The :mod:`!profile` module is deprecated and will be removed in Python 3.17. +Use :mod:`profiling.tracing` instead. -.. index:: - single: statistical profiling - single: profiling, statistical - single: deterministic profiling - single: profiling, deterministic +The :mod:`!profile` module provides a pure Python implementation of a +deterministic profiler. While useful for understanding profiler internals or +extending profiler behavior through subclassing, its pure Python implementation +introduces significant overhead compared to the C-based :mod:`profiling.tracing` +module. -Python provides both :dfn:`statistical profiling` and :dfn:`deterministic profiling` of -Python programs. A :dfn:`profile` is a set of statistics that describes how -often and for how long various parts of the program executed. These statistics -can be formatted into reports via the :mod:`pstats` module. +For most profiling tasks, use: -The Python standard library provides three different profiling implementations: +- :mod:`profiling.sampling` for production debugging with zero overhead +- :mod:`profiling.tracing` for development and testing -**Statistical Profiler:** -1. :mod:`!profiling.sampling` provides statistical profiling of running Python processes - using periodic stack sampling. It can attach to any running Python process without - requiring code modification or restart, making it ideal for production debugging. +Migration +========= -**Deterministic Profilers:** +Migrating from :mod:`!profile` to :mod:`profiling.tracing` is straightforward. +The APIs are compatible:: -2. :mod:`cProfile` is recommended for development and testing; it's a C extension with - reasonable overhead that makes it suitable for profiling long-running - programs. Based on :mod:`lsprof`, contributed by Brett Rosen and Ted - Czotter. + # Old (deprecated) + import profile + profile.run('my_function()') -3. :mod:`profile`, a pure Python module whose interface is imitated by - :mod:`cProfile`, but which adds significant overhead to profiled programs. - If you're trying to extend the profiler in some way, the task might be easier - with this module. Originally designed and written by Jim Roskind. + # New (recommended) + import profiling.tracing + profiling.tracing.run('my_function()') + +For most code, replacing ``import profile`` with ``import profiling.tracing`` +(and using ``profiling.tracing`` instead of ``profile`` throughout) provides +a straightforward migration path. .. note:: - The profiler modules are designed to provide an execution profile for a given - program, not for benchmarking purposes (for that, there is :mod:`timeit` for - reasonably accurate results). This particularly applies to benchmarking - Python code against C code: the profilers introduce overhead for Python code, - but not for C-level functions, and so the C code would seem faster than any - Python one. + The ``cProfile`` module remains available as a backward-compatible alias + to :mod:`profiling.tracing`. Existing code using ``import cProfile`` will + continue to work without modification. -**Profiler Comparison:** -+-------------------+--------------------------+----------------------+----------------------+ -| Feature | Statistical | Deterministic | Deterministic | -| | (``profiling.sampling``) | (``cProfile``) | (``profile``) | -+===================+==========================+======================+======================+ -| **Target** | Running process | Code you run | Code you run | -+-------------------+--------------------------+----------------------+----------------------+ -| **Overhead** | Virtually none | Moderate | High | -+-------------------+--------------------------+----------------------+----------------------+ -| **Accuracy** | Statistical approx. | Exact call counts | Exact call counts | -+-------------------+--------------------------+----------------------+----------------------+ -| **Setup** | Attach to any PID | Instrument code | Instrument code | -+-------------------+--------------------------+----------------------+----------------------+ -| **Use Case** | Production debugging | Development/testing | Profiler extension | -+-------------------+--------------------------+----------------------+----------------------+ -| **Implementation**| C extension | C extension | Pure Python | -+-------------------+--------------------------+----------------------+----------------------+ +:mod:`!profile` and :mod:`!profiling.tracing` module reference +============================================================== -.. note:: - - The statistical profiler (:mod:`!profiling.sampling`) is recommended for most production - use cases due to its extremely low overhead and ability to profile running processes - without modification. It can attach to any Python process and collect performance - data with minimal impact on execution speed, making it ideal for debugging - performance issues in live applications. - - -.. _statistical-profiling: - -What Is Statistical Profiling? -============================== - -:dfn:`Statistical profiling` works by periodically interrupting a running -program to capture its current call stack. Rather than monitoring every -function entry and exit like deterministic profilers, it takes snapshots at -regular intervals to build a statistical picture of where the program spends -its time. - -The sampling profiler uses process memory reading (via system calls like -``process_vm_readv`` on Linux, ``vm_read`` on macOS, and ``ReadProcessMemory`` on -Windows) to attach to a running Python process and extract stack trace -information without requiring any code modification or restart of the target -process. This approach provides several key advantages over traditional -profiling methods. - -The fundamental principle is that if a function appears frequently in the -collected stack samples, it is likely consuming significant CPU time. By -analyzing thousands of samples, the profiler can accurately estimate the -relative time spent in different parts of the program. The statistical nature -means that while individual measurements may vary, the aggregate results -converge to represent the true performance characteristics of the application. - -Since statistical profiling operates externally to the target process, it -introduces virtually no overhead to the running program. The profiler process -runs separately and reads the target process memory without interrupting its -execution. This makes it suitable for profiling production systems where -performance impact must be minimized. - -The accuracy of statistical profiling improves with the number of samples -collected. Short-lived functions may be missed or underrepresented, while -long-running functions will be captured proportionally to their execution time. -This characteristic makes statistical profiling particularly effective for -identifying the most significant performance bottlenecks rather than providing -exhaustive coverage of all function calls. - -Statistical profiling excels at answering questions like "which functions -consume the most CPU time?" and "where should I focus optimization efforts?" -rather than "exactly how many times was this function called?" The trade-off -between precision and practicality makes it an invaluable tool for performance -analysis in real-world applications. - -.. _profile-instant: - -Instant User's Manual -===================== - -This section is provided for users that "don't want to read the manual." It -provides a very brief overview, and allows a user to rapidly perform profiling -on an existing application. - -**Statistical Profiling (Recommended for Production):** - -To profile an existing running process:: - - python -m profiling.sampling 1234 - -To profile with custom settings:: - - python -m profiling.sampling -i 50 -d 30 1234 - -**Deterministic Profiling (Development/Testing):** - -To profile a function that takes a single argument, you can do:: - - import cProfile - import re - cProfile.run('re.compile("foo|bar")') - -(Use :mod:`profile` instead of :mod:`cProfile` if the latter is not available on -your system.) - -The above action would run :func:`re.compile` and print profile results like -the following:: - - 214 function calls (207 primitive calls) in 0.002 seconds - - Ordered by: cumulative time - - ncalls tottime percall cumtime percall filename:lineno(function) - 1 0.000 0.000 0.002 0.002 {built-in method builtins.exec} - 1 0.000 0.000 0.001 0.001 :1() - 1 0.000 0.000 0.001 0.001 __init__.py:250(compile) - 1 0.000 0.000 0.001 0.001 __init__.py:289(_compile) - 1 0.000 0.000 0.000 0.000 _compiler.py:759(compile) - 1 0.000 0.000 0.000 0.000 _parser.py:937(parse) - 1 0.000 0.000 0.000 0.000 _compiler.py:598(_code) - 1 0.000 0.000 0.000 0.000 _parser.py:435(_parse_sub) - -The first line indicates that 214 calls were monitored. Of those calls, 207 -were :dfn:`primitive`, meaning that the call was not induced via recursion. The -next line: ``Ordered by: cumulative time`` indicates the output is sorted -by the ``cumtime`` values. The column headings include: - -ncalls - for the number of calls. - -tottime - for the total time spent in the given function (and excluding time made in - calls to sub-functions) - -percall - is the quotient of ``tottime`` divided by ``ncalls`` - -cumtime - is the cumulative time spent in this and all subfunctions (from invocation - till exit). This figure is accurate *even* for recursive functions. - -percall - is the quotient of ``cumtime`` divided by primitive calls - -filename:lineno(function) - provides the respective data of each function - -When there are two numbers in the first column (for example ``3/1``), it means -that the function recursed. The second value is the number of primitive calls -and the former is the total number of calls. Note that when the function does -not recurse, these two values are the same, and only the single figure is -printed. - -Instead of printing the output at the end of the profile run, you can save the -results to a file by specifying a filename to the :func:`run` function:: - - import cProfile - import re - cProfile.run('re.compile("foo|bar")', 'restats') - -The :class:`pstats.Stats` class reads profile results from a file and formats -them in various ways. - -.. _sampling-profiler-cli: - -Statistical Profiler Command Line Interface -=========================================== - -.. program:: profiling.sampling - -The :mod:`!profiling.sampling` module can be invoked as a script to profile running processes:: - - python -m profiling.sampling [options] PID - -**Basic Usage Examples:** - -Profile process 1234 for 10 seconds with default settings:: - - python -m profiling.sampling 1234 - -Profile with custom interval and duration, save to file:: - - python -m profiling.sampling -i 50 -d 30 -o profile.stats 1234 - -Generate collapsed stacks to use with tools like `flamegraph.pl -`_:: - - python -m profiling.sampling --collapsed 1234 - -Profile all threads, sort by total time:: - - python -m profiling.sampling -a --sort-tottime 1234 - -Profile with real-time sampling statistics:: - - python -m profiling.sampling --realtime-stats 1234 - -**Command Line Options:** - -.. option:: PID - - Process ID of the Python process to profile (required) - -.. option:: -i, --interval INTERVAL - - Sampling interval in microseconds (default: 100) - -.. option:: -d, --duration DURATION - - Sampling duration in seconds (default: 10) - -.. option:: -a, --all-threads - - Sample all threads in the process instead of just the main thread - -.. option:: --realtime-stats - - Print real-time sampling statistics during profiling - -.. option:: --pstats - - Generate pstats output (default) - -.. option:: --collapsed - - Generate collapsed stack traces for flamegraphs - -.. option:: -o, --outfile OUTFILE - - Save output to a file - -**Sorting Options (pstats format only):** - -.. option:: --sort-nsamples - - Sort by number of direct samples - -.. option:: --sort-tottime - - Sort by total time - -.. option:: --sort-cumtime - - Sort by cumulative time (default) - -.. option:: --sort-sample-pct - - Sort by sample percentage - -.. option:: --sort-cumul-pct - - Sort by cumulative sample percentage - -.. option:: --sort-nsamples-cumul - - Sort by cumulative samples - -.. option:: --sort-name - - Sort by function name - -.. option:: -l, --limit LIMIT - - Limit the number of rows in the output (default: 15) - -.. option:: --no-summary - - Disable the summary section in the output - -**Understanding Statistical Profile Output:** - -The statistical profiler produces output similar to deterministic profilers but with different column meanings:: - - Profile Stats: - nsamples sample% tottime (ms) cumul% cumtime (ms) filename:lineno(function) - 45/67 12.5 23.450 18.6 56.780 mymodule.py:42(process_data) - 23/23 6.4 15.230 6.4 15.230 :0(len) - -**Column Meanings:** - -- **nsamples**: ``direct/cumulative`` - Times function was directly executing / on call stack -- **sample%**: Percentage of total samples where function was directly executing -- **tottime**: Estimated time spent directly in this function -- **cumul%**: Percentage of samples where function was anywhere on call stack -- **cumtime**: Estimated cumulative time including called functions -- **filename:lineno(function)**: Location and name of the function - -.. _profile-cli: - -:mod:`!profiling.sampling` Module Reference -======================================================= - -.. module:: profiling.sampling - :synopsis: Python statistical profiler. - -This section documents the programmatic interface for the :mod:`!profiling.sampling` module. -For command-line usage, see :ref:`sampling-profiler-cli`. For conceptual information -about statistical profiling, see :ref:`statistical-profiling` - -.. function:: sample(pid, *, sort=2, sample_interval_usec=100, duration_sec=10, filename=None, all_threads=False, limit=None, show_summary=True, output_format="pstats", realtime_stats=False) - - Sample a Python process and generate profiling data. - - This is the main entry point for statistical profiling. It creates a - :class:`SampleProfiler`, collects stack traces from the target process, and - outputs the results in the specified format. - - :param int pid: Process ID of the target Python process - :param int sort: Sort order for pstats output (default: 2 for cumulative time) - :param int sample_interval_usec: Sampling interval in microseconds (default: 100) - :param int duration_sec: Duration to sample in seconds (default: 10) - :param str filename: Output filename (None for stdout/default naming) - :param bool all_threads: Whether to sample all threads (default: False) - :param int limit: Maximum number of functions to display (default: None) - :param bool show_summary: Whether to show summary statistics (default: True) - :param str output_format: Output format - 'pstats' or 'collapsed' (default: 'pstats') - :param bool realtime_stats: Whether to display real-time statistics (default: False) - - :raises ValueError: If output_format is not 'pstats' or 'collapsed' - - Examples:: - - # Basic usage - profile process 1234 for 10 seconds - import profiling.sampling - profiling.sampling.sample(1234) - - # Profile with custom settings - profiling.sampling.sample(1234, duration_sec=30, sample_interval_usec=50, all_threads=True) - - # Generate collapsed stack traces for flamegraph.pl - profiling.sampling.sample(1234, output_format='collapsed', filename='profile.collapsed') - -.. class:: SampleProfiler(pid, sample_interval_usec, all_threads) - - Low-level API for the statistical profiler. - - This profiler uses periodic stack sampling to collect performance data - from running Python processes with minimal overhead. It can attach to - any Python process by PID and collect stack traces at regular intervals. - - :param int pid: Process ID of the target Python process - :param int sample_interval_usec: Sampling interval in microseconds - :param bool all_threads: Whether to sample all threads or just the main thread - - .. method:: sample(collector, duration_sec=10) - - Sample the target process for the specified duration. - - Collects stack traces from the target process at regular intervals - and passes them to the provided collector for processing. - - :param collector: Object that implements ``collect()`` method to process stack traces - :param int duration_sec: Duration to sample in seconds (default: 10) - - The method tracks sampling statistics and can display real-time - information if realtime_stats is enabled. - -.. seealso:: - - :ref:`sampling-profiler-cli` - Command-line interface documentation for the statistical profiler. - -Deterministic Profiler Command Line Interface -============================================= - -.. program:: cProfile - -The files :mod:`cProfile` and :mod:`profile` can also be invoked as a script to -profile another script. For example:: - - python -m cProfile [-o output_file] [-s sort_order] (-m module | myscript.py) - -.. option:: -o - - Writes the profile results to a file instead of to stdout. - -.. option:: -s - - Specifies one of the :func:`~pstats.Stats.sort_stats` sort values - to sort the output by. - This only applies when :option:`-o ` is not supplied. - -.. option:: -m - - Specifies that a module is being profiled instead of a script. - - .. versionadded:: 3.7 - Added the ``-m`` option to :mod:`cProfile`. - - .. versionadded:: 3.8 - Added the ``-m`` option to :mod:`profile`. - -The :mod:`pstats` module's :class:`~pstats.Stats` class has a variety of methods -for manipulating and printing the data saved into a profile results file:: - - import pstats - from pstats import SortKey - p = pstats.Stats('restats') - p.strip_dirs().sort_stats(-1).print_stats() - -The :meth:`~pstats.Stats.strip_dirs` method removed the extraneous path from all -the module names. The :meth:`~pstats.Stats.sort_stats` method sorted all the -entries according to the standard module/line/name string that is printed. The -:meth:`~pstats.Stats.print_stats` method printed out all the statistics. You -might try the following sort calls:: - - p.sort_stats(SortKey.NAME) - p.print_stats() - -The first call will actually sort the list by function name, and the second call -will print out the statistics. The following are some interesting calls to -experiment with:: - - p.sort_stats(SortKey.CUMULATIVE).print_stats(10) - -This sorts the profile by cumulative time in a function, and then only prints -the ten most significant lines. If you want to understand what algorithms are -taking time, the above line is what you would use. - -If you were looking to see what functions were looping a lot, and taking a lot -of time, you would do:: - - p.sort_stats(SortKey.TIME).print_stats(10) - -to sort according to time spent within each function, and then print the -statistics for the top ten functions. - -You might also try:: - - p.sort_stats(SortKey.FILENAME).print_stats('__init__') - -This will sort all the statistics by file name, and then print out statistics -for only the class init methods (since they are spelled with ``__init__`` in -them). As one final example, you could try:: - - p.sort_stats(SortKey.TIME, SortKey.CUMULATIVE).print_stats(.5, 'init') - -This line sorts statistics with a primary key of time, and a secondary key of -cumulative time, and then prints out some of the statistics. To be specific, the -list is first culled down to 50% (re: ``.5``) of its original size, then only -lines containing ``init`` are maintained, and that sub-sub-list is printed. - -If you wondered what functions called the above functions, you could now (``p`` -is still sorted according to the last criteria) do:: - - p.print_callers(.5, 'init') - -and you would get a list of callers for each of the listed functions. - -If you want more functionality, you're going to have to read the manual, or -guess what the following functions do:: - - p.print_callees() - p.add('restats') - -Invoked as a script, the :mod:`pstats` module is a statistics browser for -reading and examining profile dumps. It has a simple line-oriented interface -(implemented using :mod:`cmd`) and interactive help. - -:mod:`profile` and :mod:`cProfile` Module Reference -======================================================= - -.. module:: cProfile -.. module:: profile - :synopsis: Python source profiler. - -Both the :mod:`profile` and :mod:`cProfile` modules provide the following -functions: +Both the :mod:`!profile` and :mod:`profiling.tracing` modules provide the +following functions: .. function:: run(command, filename=None, sort=-1) @@ -545,7 +86,7 @@ functions: .. class:: Profile(timer=None, timeunit=0.0, subcalls=True, builtins=True) This class is normally only used if more precise control over profiling is - needed than what the :func:`cProfile.run` function provides. + needed than what the :func:`profiling.tracing.run` function provides. A custom timer can be supplied for measuring how long code takes to run via the *timer* argument. This must be a function that returns a single number @@ -557,9 +98,12 @@ functions: Directly using the :class:`Profile` class allows formatting profile results without writing the profile data to a file:: - import cProfile, pstats, io + import profiling.tracing + import pstats + import io from pstats import SortKey - pr = cProfile.Profile() + + pr = profiling.tracing.Profile() pr.enable() # ... do something ... pr.disable() @@ -570,11 +114,12 @@ functions: print(s.getvalue()) The :class:`Profile` class can also be used as a context manager (supported - only in :mod:`cProfile` module. see :ref:`typecontextmanager`):: + only in :mod:`profiling.tracing`, not in the deprecated :mod:`!profile` + module; see :ref:`typecontextmanager`):: - import cProfile + import profiling.tracing - with cProfile.Profile() as pr: + with profiling.tracing.Profile() as pr: # ... do something ... pr.print_stats() @@ -584,11 +129,11 @@ functions: .. method:: enable() - Start collecting profiling data. Only in :mod:`cProfile`. + Start collecting profiling data. Only in :mod:`profiling.tracing`. .. method:: disable() - Stop collecting profiling data. Only in :mod:`cProfile`. + Stop collecting profiling data. Only in :mod:`profiling.tracing`. .. method:: create_stats() @@ -602,7 +147,7 @@ functions: The *sort* parameter specifies the sorting order of the displayed statistics. It accepts a single key or a tuple of keys to enable - multi-level sorting, as in :func:`Stats.sort_stats `. + multi-level sorting, as in :meth:`pstats.Stats.sort_stats`. .. versionadded:: 3.13 :meth:`~Profile.print_stats` now accepts a tuple of keys. @@ -629,237 +174,43 @@ returns. If the interpreter is terminated (e.g. via a :func:`sys.exit` call during the called command/function execution) no profiling results will be printed. -.. _profile-stats: -The :class:`Stats` Class -======================== +Differences from :mod:`!profiling.tracing` +========================================== -Analysis of the profiler data is done using the :class:`~pstats.Stats` class. +The :mod:`!profile` module differs from :mod:`profiling.tracing` in several +ways: -.. module:: pstats - :synopsis: Statistics object for use with the profiler. +**Higher overhead.** The pure Python implementation is significantly slower +than the C implementation, making it unsuitable for profiling long-running +programs or performance-sensitive code. -.. class:: Stats(*filenames or profile, stream=sys.stdout) +**Calibration support.** The :mod:`!profile` module supports calibration to +compensate for profiling overhead. This is not needed in :mod:`profiling.tracing` +because the C implementation has negligible overhead. - This class constructor creates an instance of a "statistics object" from a - *filename* (or list of filenames) or from a :class:`Profile` instance. Output - will be printed to the stream specified by *stream*. +**Custom timers.** Both modules support custom timers, but :mod:`!profile` +accepts timer functions that return tuples (like :func:`os.times`), while +:mod:`profiling.tracing` requires a function returning a single number. - The file selected by the above constructor must have been created by the - corresponding version of :mod:`profile` or :mod:`cProfile`. To be specific, - there is *no* file compatibility guaranteed with future versions of this - profiler, and there is no compatibility with files produced by other - profilers, or the same profiler run on a different operating system. If - several files are provided, all the statistics for identical functions will - be coalesced, so that an overall view of several processes can be considered - in a single report. If additional files need to be combined with data in an - existing :class:`~pstats.Stats` object, the :meth:`~pstats.Stats.add` method - can be used. +**Subclassing.** The pure Python implementation is easier to subclass and +extend for custom profiling behavior. - Instead of reading the profile data from a file, a :class:`cProfile.Profile` - or :class:`profile.Profile` object can be used as the profile data source. - - :class:`Stats` objects have the following methods: - - .. method:: strip_dirs() - - This method for the :class:`Stats` class removes all leading path - information from file names. It is very useful in reducing the size of - the printout to fit within (close to) 80 columns. This method modifies - the object, and the stripped information is lost. After performing a - strip operation, the object is considered to have its entries in a - "random" order, as it was just after object initialization and loading. - If :meth:`~pstats.Stats.strip_dirs` causes two function names to be - indistinguishable (they are on the same line of the same filename, and - have the same function name), then the statistics for these two entries - are accumulated into a single entry. - - - .. method:: add(*filenames) - - This method of the :class:`Stats` class accumulates additional profiling - information into the current profiling object. Its arguments should refer - to filenames created by the corresponding version of :func:`profile.run` - or :func:`cProfile.run`. Statistics for identically named (re: file, line, - name) functions are automatically accumulated into single function - statistics. - - - .. method:: dump_stats(filename) - - Save the data loaded into the :class:`Stats` object to a file named - *filename*. The file is created if it does not exist, and is overwritten - if it already exists. This is equivalent to the method of the same name - on the :class:`profile.Profile` and :class:`cProfile.Profile` classes. - - - .. method:: sort_stats(*keys) - - This method modifies the :class:`Stats` object by sorting it according to - the supplied criteria. The argument can be either a string or a SortKey - enum identifying the basis of a sort (example: ``'time'``, ``'name'``, - ``SortKey.TIME`` or ``SortKey.NAME``). The SortKey enums argument have - advantage over the string argument in that it is more robust and less - error prone. - - When more than one key is provided, then additional keys are used as - secondary criteria when there is equality in all keys selected before - them. For example, ``sort_stats(SortKey.NAME, SortKey.FILE)`` will sort - all the entries according to their function name, and resolve all ties - (identical function names) by sorting by file name. - - For the string argument, abbreviations can be used for any key names, as - long as the abbreviation is unambiguous. - - The following are the valid string and SortKey: - - +------------------+---------------------+----------------------+ - | Valid String Arg | Valid enum Arg | Meaning | - +==================+=====================+======================+ - | ``'calls'`` | SortKey.CALLS | call count | - +------------------+---------------------+----------------------+ - | ``'cumulative'`` | SortKey.CUMULATIVE | cumulative time | - +------------------+---------------------+----------------------+ - | ``'cumtime'`` | N/A | cumulative time | - +------------------+---------------------+----------------------+ - | ``'file'`` | N/A | file name | - +------------------+---------------------+----------------------+ - | ``'filename'`` | SortKey.FILENAME | file name | - +------------------+---------------------+----------------------+ - | ``'module'`` | N/A | file name | - +------------------+---------------------+----------------------+ - | ``'ncalls'`` | N/A | call count | - +------------------+---------------------+----------------------+ - | ``'pcalls'`` | SortKey.PCALLS | primitive call count | - +------------------+---------------------+----------------------+ - | ``'line'`` | SortKey.LINE | line number | - +------------------+---------------------+----------------------+ - | ``'name'`` | SortKey.NAME | function name | - +------------------+---------------------+----------------------+ - | ``'nfl'`` | SortKey.NFL | name/file/line | - +------------------+---------------------+----------------------+ - | ``'stdname'`` | SortKey.STDNAME | standard name | - +------------------+---------------------+----------------------+ - | ``'time'`` | SortKey.TIME | internal time | - +------------------+---------------------+----------------------+ - | ``'tottime'`` | N/A | internal time | - +------------------+---------------------+----------------------+ - - Note that all sorts on statistics are in descending order (placing most - time consuming items first), where as name, file, and line number searches - are in ascending order (alphabetical). The subtle distinction between - ``SortKey.NFL`` and ``SortKey.STDNAME`` is that the standard name is a - sort of the name as printed, which means that the embedded line numbers - get compared in an odd way. For example, lines 3, 20, and 40 would (if - the file names were the same) appear in the string order 20, 3 and 40. - In contrast, ``SortKey.NFL`` does a numeric compare of the line numbers. - In fact, ``sort_stats(SortKey.NFL)`` is the same as - ``sort_stats(SortKey.NAME, SortKey.FILENAME, SortKey.LINE)``. - - For backward-compatibility reasons, the numeric arguments ``-1``, ``0``, - ``1``, and ``2`` are permitted. They are interpreted as ``'stdname'``, - ``'calls'``, ``'time'``, and ``'cumulative'`` respectively. If this old - style format (numeric) is used, only one sort key (the numeric key) will - be used, and additional arguments will be silently ignored. - - .. For compatibility with the old profiler. - - .. versionadded:: 3.7 - Added the SortKey enum. - - .. method:: reverse_order() - - This method for the :class:`Stats` class reverses the ordering of the - basic list within the object. Note that by default ascending vs - descending order is properly selected based on the sort key of choice. - - .. This method is provided primarily for compatibility with the old - profiler. - - - .. method:: print_stats(*restrictions) - - This method for the :class:`Stats` class prints out a report as described - in the :func:`profile.run` definition. - - The order of the printing is based on the last - :meth:`~pstats.Stats.sort_stats` operation done on the object (subject to - caveats in :meth:`~pstats.Stats.add` and - :meth:`~pstats.Stats.strip_dirs`). - - The arguments provided (if any) can be used to limit the list down to the - significant entries. Initially, the list is taken to be the complete set - of profiled functions. Each restriction is either an integer (to select a - count of lines), or a decimal fraction between 0.0 and 1.0 inclusive (to - select a percentage of lines), or a string that will be interpreted as a - regular expression (to pattern match the standard name that is printed). - If several restrictions are provided, then they are applied sequentially. - For example:: - - print_stats(.1, 'foo:') - - would first limit the printing to first 10% of list, and then only print - functions that were part of filename :file:`.\*foo:`. In contrast, the - command:: - - print_stats('foo:', .1) - - would limit the list to all functions having file names :file:`.\*foo:`, - and then proceed to only print the first 10% of them. - - - .. method:: print_callers(*restrictions) - - This method for the :class:`Stats` class prints a list of all functions - that called each function in the profiled database. The ordering is - identical to that provided by :meth:`~pstats.Stats.print_stats`, and the - definition of the restricting argument is also identical. Each caller is - reported on its own line. The format differs slightly depending on the - profiler that produced the stats: - - * With :mod:`profile`, a number is shown in parentheses after each caller - to show how many times this specific call was made. For convenience, a - second non-parenthesized number repeats the cumulative time spent in the - function at the right. - - * With :mod:`cProfile`, each caller is preceded by three numbers: the - number of times this specific call was made, and the total and - cumulative times spent in the current function while it was invoked by - this specific caller. - - - .. method:: print_callees(*restrictions) - - This method for the :class:`Stats` class prints a list of all function - that were called by the indicated function. Aside from this reversal of - direction of calls (re: called vs was called by), the arguments and - ordering are identical to the :meth:`~pstats.Stats.print_callers` method. - - - .. method:: get_stats_profile() - - This method returns an instance of StatsProfile, which contains a mapping - of function names to instances of FunctionProfile. Each FunctionProfile - instance holds information related to the function's profile such as how - long the function took to run, how many times it was called, etc... - - .. versionadded:: 3.9 - Added the following dataclasses: StatsProfile, FunctionProfile. - Added the following function: get_stats_profile. .. _deterministic-profiling: -What Is Deterministic Profiling? +What is deterministic profiling? ================================ :dfn:`Deterministic profiling` is meant to reflect the fact that all *function call*, *function return*, and *exception* events are monitored, and precise timings are made for the intervals between these events (during which time the user's code is executing). In contrast, :dfn:`statistical profiling` (which is -provided by the :mod:`!profiling.sampling` module) periodically samples the effective instruction pointer, and -deduces where time is being spent. The latter technique traditionally involves -less overhead (as the code does not need to be instrumented), but provides only -relative indications of where time is being spent. +provided by the :mod:`profiling.sampling` module) periodically samples the +effective instruction pointer, and deduces where time is being spent. The +latter technique traditionally involves less overhead (as the code does not +need to be instrumented), but provides only relative indications of where time +is being spent. In Python, since there is an interpreter active during execution, the presence of instrumented code is not required in order to do deterministic profiling. @@ -903,15 +254,16 @@ this error. The error that accumulates in this fashion is typically less than the accuracy of the clock (less than one clock tick), but it *can* accumulate and become very significant. -The problem is more important with :mod:`profile` than with the lower-overhead -:mod:`cProfile`. For this reason, :mod:`profile` provides a means of -calibrating itself for a given platform so that this error can be -probabilistically (on the average) removed. After the profiler is calibrated, it -will be more accurate (in a least square sense), but it will sometimes produce -negative numbers (when call counts are exceptionally low, and the gods of -probability work against you :-). ) Do *not* be alarmed by negative numbers in -the profile. They should *only* appear if you have calibrated your profiler, -and the results are actually better than without calibration. +The problem is more important with the deprecated :mod:`!profile` module than +with the lower-overhead :mod:`profiling.tracing`. For this reason, +:mod:`!profile` provides a means of calibrating itself for a given platform so +that this error can be probabilistically (on the average) removed. After the +profiler is calibrated, it will be more accurate (in a least square sense), but +it will sometimes produce negative numbers (when call counts are exceptionally +low, and the gods of probability work against you :-). ) Do *not* be alarmed +by negative numbers in the profile. They should *only* appear if you have +calibrated your profiler, and the results are actually better than without +calibration. .. _profile-calibration: @@ -919,7 +271,7 @@ and the results are actually better than without calibration. Calibration =========== -The profiler of the :mod:`profile` module subtracts a constant from each event +The profiler of the :mod:`!profile` module subtracts a constant from each event handling time to compensate for the overhead of calling the time function, and socking away the results. By default, the constant is 0. The following procedure can be used to obtain a better constant for a given platform (see @@ -957,6 +309,7 @@ When you have a consistent answer, there are three ways you can use it:: If you have a choice, you are better off choosing a smaller constant, and then your results will "less often" show up as negative in profile statistics. + .. _profile-timers: Using a custom timer @@ -969,7 +322,7 @@ to the :class:`Profile` class constructor:: pr = profile.Profile(your_time_func) The resulting profiler will then call ``your_time_func``. Depending on whether -you are using :class:`profile.Profile` or :class:`cProfile.Profile`, +you are using :class:`profile.Profile` or :class:`profiling.tracing.Profile`, ``your_time_func``'s return value will be interpreted differently: :class:`profile.Profile` @@ -988,20 +341,32 @@ you are using :class:`profile.Profile` or :class:`cProfile.Profile`, replacement dispatch method that best handles your timer call, along with the appropriate calibration constant. -:class:`cProfile.Profile` +:class:`profiling.tracing.Profile` ``your_time_func`` should return a single number. If it returns integers, you can also invoke the class constructor with a second argument specifying the real duration of one unit of time. For example, if ``your_integer_time_func`` returns times measured in thousands of seconds, you would construct the :class:`Profile` instance as follows:: - pr = cProfile.Profile(your_integer_time_func, 0.001) + pr = profiling.tracing.Profile(your_integer_time_func, 0.001) - As the :class:`cProfile.Profile` class cannot be calibrated, custom timer - functions should be used with care and should be as fast as possible. For - the best results with a custom timer, it might be necessary to hard-code it - in the C source of the internal :mod:`!_lsprof` module. + As the :class:`profiling.tracing.Profile` class cannot be calibrated, custom + timer functions should be used with care and should be as fast as possible. + For the best results with a custom timer, it might be necessary to hard-code + it in the C source of the internal :mod:`!_lsprof` module. Python 3.3 adds several new functions in :mod:`time` that can be used to make precise measurements of process or wall-clock time. For example, see :func:`time.perf_counter`. + + +.. seealso:: + + :mod:`profiling` + Overview of Python profiling tools. + + :mod:`profiling.tracing` + Recommended replacement for this module. + + :mod:`pstats` + Statistical analysis and formatting for profile data. diff --git a/Doc/library/profiling-sampling-visualization.html b/Doc/library/profiling-sampling-visualization.html new file mode 100644 index 00000000000..0cbd0f2374d --- /dev/null +++ b/Doc/library/profiling-sampling-visualization.html @@ -0,0 +1 @@ +
diff --git a/Doc/library/profiling.rst b/Doc/library/profiling.rst new file mode 100644 index 00000000000..f4ac47826b2 --- /dev/null +++ b/Doc/library/profiling.rst @@ -0,0 +1,270 @@ +.. highlight:: sh + +.. _profiling-module: + +************************************** +:mod:`!profiling` --- Python profilers +************************************** + +.. module:: profiling + :synopsis: Python profiling tools for performance analysis. + +.. versionadded:: 3.15 + +**Source code:** :source:`Lib/profiling/` + +-------------- + +.. index:: + single: statistical profiling + single: profiling, statistical + single: deterministic profiling + single: profiling, deterministic + + +Introduction to profiling +========================= + +A :dfn:`profile` is a set of statistics that describes how often and for how +long various parts of a program execute. These statistics help identify +performance bottlenecks and guide optimization efforts. Python provides two +fundamentally different approaches to collecting this information: statistical +sampling and deterministic tracing. + +The :mod:`!profiling` package organizes Python's built-in profiling tools under +a single namespace. It contains two submodules, each implementing a different +profiling methodology: + +:mod:`profiling.sampling` + A statistical profiler that periodically samples the call stack. Run scripts + directly or attach to running processes by PID. Provides multiple output + formats (flame graphs, heatmaps, Firefox Profiler), GIL analysis, GC tracking, + and multiple profiling modes (wall-clock, CPU, GIL) with virtually no overhead. + +:mod:`profiling.tracing` + A deterministic profiler that traces every function call, return, and + exception event. Provides exact call counts and precise timing information, + capturing every invocation including very fast functions. + +.. note:: + + The profiler modules are designed to provide an execution profile for a + given program, not for benchmarking purposes. For benchmarking, use the + :mod:`timeit` module, which provides reasonably accurate timing + measurements. This distinction is particularly important when comparing + Python code against C code: deterministic profilers introduce overhead for + Python code but not for C-level functions, which can skew comparisons. + + +.. _choosing-a-profiler: + +Choosing a profiler +=================== + +For most performance analysis, use the statistical profiler +(:mod:`profiling.sampling`). It has minimal overhead, works for both development +and production, and provides rich visualization options including flame graphs, +heatmaps, GIL analysis, and more. + +Use the deterministic profiler (:mod:`profiling.tracing`) when you need **exact +call counts** and cannot afford to miss any function calls. Since it instruments +every function call and return, it will capture even very fast functions that +complete between sampling intervals. The tradeoff is higher overhead. + +The following table summarizes the key differences: + ++--------------------+------------------------------+------------------------------+ +| Feature | Statistical sampling | Deterministic | +| | (:mod:`profiling.sampling`) | (:mod:`profiling.tracing`) | ++====================+==============================+==============================+ +| **Overhead** | Virtually none | Moderate | ++--------------------+------------------------------+------------------------------+ +| **Accuracy** | Statistical estimate | Exact call counts | ++--------------------+------------------------------+------------------------------+ +| **Output formats** | pstats, flame graph, heatmap,| pstats | +| | gecko, collapsed | | ++--------------------+------------------------------+------------------------------+ +| **Profiling modes**| Wall-clock, CPU, GIL | Wall-clock | ++--------------------+------------------------------+------------------------------+ +| **Special frames** | GC, native (C extensions) | N/A | ++--------------------+------------------------------+------------------------------+ +| **Attach to PID** | Yes | No | ++--------------------+------------------------------+------------------------------+ + + +When to use statistical sampling +-------------------------------- + +The statistical profiler (:mod:`profiling.sampling`) is recommended for most +performance analysis tasks. Use it the same way you would use +:mod:`profiling.tracing`:: + + python -m profiling.sampling run script.py + +One of the main strengths of the sampling profiler is its variety of output +formats. Beyond traditional pstats tables, it can generate interactive +flame graphs that visualize call hierarchies, line-level source heatmaps that +show exactly where time is spent in your code, and Firefox Profiler output for +timeline-based analysis. + +The profiler also provides insight into Python interpreter behavior that +deterministic profiling cannot capture. Use ``--mode gil`` to identify GIL +contention in multi-threaded code, ``--mode cpu`` to measure actual CPU time +excluding I/O waits, or inspect ```` frames to understand garbage collection +overhead. The ``--native`` option reveals time spent in C extensions, helping +distinguish Python overhead from library performance. + +For multi-threaded applications, the ``-a`` option samples all threads +simultaneously, showing how work is distributed. And for production debugging, +the ``attach`` command connects to any running Python process by PID without +requiring a restart or code changes. + + +When to use deterministic tracing +--------------------------------- + +The deterministic profiler (:mod:`profiling.tracing`) instruments every function +call and return. This approach has higher overhead than sampling, but guarantees +complete coverage of program execution. + +The primary reason to choose deterministic tracing is when you need exact call +counts. Statistical profiling estimates frequency based on sampling, which may +undercount short-lived functions that complete between samples. If you need to +verify that an optimization actually reduced the number of function calls, or +if you want to trace the complete call graph to understand caller-callee +relationships, deterministic tracing is the right choice. + +Deterministic tracing also excels at capturing functions that execute in +microseconds. Such functions may not appear frequently enough in statistical +samples, but deterministic tracing records every invocation regardless of +duration. + + +Quick start +=========== + +This section provides the minimal steps needed to start profiling. For complete +documentation, see the dedicated pages for each profiler. + + +Statistical profiling +--------------------- + +To profile a script, use the :mod:`profiling.sampling` module with the ``run`` +command:: + + python -m profiling.sampling run script.py + python -m profiling.sampling run -m mypackage.module + +This runs the script under the profiler and prints a summary of where time was +spent. For an interactive flame graph:: + + python -m profiling.sampling run --flamegraph script.py + +To profile an already-running process, use the ``attach`` command with the +process ID:: + + python -m profiling.sampling attach 1234 + +For custom settings, specify the sampling interval (in microseconds) and +duration (in seconds):: + + python -m profiling.sampling run -i 50 -d 30 script.py + + +Deterministic profiling +----------------------- + +To profile a script from the command line:: + + python -m profiling.tracing script.py + +To profile a piece of code programmatically: + +.. code-block:: python + + import profiling.tracing + profiling.tracing.run('my_function()') + +This executes the given code under the profiler and prints a summary showing +exact function call counts and timing. + + +.. _profile-output: + +Understanding profile output +============================ + +Both profilers collect function-level statistics, though they present them in +different formats. The sampling profiler offers multiple visualizations +(flame graphs, heatmaps, Firefox Profiler, pstats tables), while the +deterministic profiler produces pstats-compatible output. Regardless of format, +the underlying concepts are the same. + +Key profiling concepts: + +**Direct time** (also called *self time* or *tottime*) + Time spent executing code in the function itself, excluding time spent in + functions it called. High direct time indicates the function contains + expensive operations. + +**Cumulative time** (also called *total time* or *cumtime*) + Time spent in the function and all functions it called. This measures the + total cost of calling a function, including its entire call subtree. + +**Call count** (also called *ncalls* or *samples*) + How many times the function was called (deterministic) or sampled + (statistical). In deterministic profiling, this is exact. In statistical + profiling, it represents the number of times the function appeared in a + stack sample. + +**Primitive calls** + Calls that are not induced by recursion. When a function recurses, the total + call count includes recursive invocations, but primitive calls counts only + the initial entry. Displayed as ``total/primitive`` (for example, ``3/1`` + means three total calls, one primitive). + +**Caller/Callee relationships** + Which functions called a given function (callers) and which functions it + called (callees). Flame graphs visualize this as nested rectangles; pstats + can display it via the :meth:`~pstats.Stats.print_callers` and + :meth:`~pstats.Stats.print_callees` methods. + + +Legacy compatibility +==================== + +For backward compatibility, the ``cProfile`` module remains available as an +alias to :mod:`profiling.tracing`. Existing code using ``import cProfile`` will +continue to work without modification in all future Python versions. + +.. deprecated:: 3.15 + + The pure Python :mod:`profile` module is deprecated and will be removed in + Python 3.17. Use :mod:`profiling.tracing` (or its alias ``cProfile``) + instead. See :mod:`profile` for migration guidance. + + +.. seealso:: + + :mod:`profiling.sampling` + Statistical sampling profiler with flame graphs, heatmaps, and GIL analysis. + Recommended for most users. + + :mod:`profiling.tracing` + Deterministic tracing profiler for exact call counts. + + :mod:`pstats` + Statistics analysis and formatting for profile data. + + :mod:`timeit` + Module for measuring execution time of small code snippets. + + +.. rubric:: Submodules + +.. toctree:: + :maxdepth: 1 + + profiling.tracing.rst + profiling.sampling.rst diff --git a/Doc/library/profiling.sampling.rst b/Doc/library/profiling.sampling.rst new file mode 100644 index 00000000000..790d3600180 --- /dev/null +++ b/Doc/library/profiling.sampling.rst @@ -0,0 +1,1617 @@ +.. highlight:: sh + +.. _profiling-sampling: + +*************************************************** +:mod:`!profiling.sampling` --- Statistical profiler +*************************************************** + +.. module:: profiling.sampling + :synopsis: Statistical sampling profiler for Python processes. + +.. versionadded:: 3.15 + +**Source code:** :source:`Lib/profiling/sampling/` + +.. program:: profiling.sampling + +-------------- + +.. image:: ../../Lib/profiling/sampling/_assets/tachyon-logo.png + :alt: Tachyon logo + :align: center + :width: 300px + +The :mod:`!profiling.sampling` module, named **Tachyon**, provides statistical +profiling of Python programs through periodic stack sampling. Tachyon can +run scripts directly or attach to any running Python process without requiring +code changes or restarts. Because sampling occurs externally to the target +process, overhead is virtually zero, making Tachyon suitable for both +development and production environments. + + +What is statistical profiling? +============================== + +Statistical profiling builds a picture of program behavior by periodically +capturing snapshots of the call stack. Rather than instrumenting every function +call and return as deterministic profilers do, Tachyon reads the call stack at +regular intervals to record what code is currently running. + +This approach rests on a simple principle: functions that consume significant +CPU time will appear frequently in the collected samples. By gathering thousands +of samples over a profiling session, Tachyon constructs an accurate statistical +estimate of where time is spent. The more samples collected, the +more precise this estimate becomes. + +.. only:: html + + The following interactive visualization demonstrates how sampling profiling + works. Press **Play** to watch a Python program execute, and observe how the + profiler periodically captures snapshots of the call stack. Adjust the + **sample interval** to see how sampling frequency affects the results. + + .. raw:: html + :file: profiling-sampling-visualization.html + +.. only:: not html + + .. note:: + + An interactive visualization of sampling profiling is available in the + HTML version of this documentation. + + +How time is estimated +--------------------- + +The time values shown in Tachyon's output are **estimates derived from sample +counts**, not direct measurements. Tachyon counts how many times each function +appears in the collected samples, then multiplies by the sampling interval to +estimate time. + +For example, with a 10 kHz sampling rate over a 10-second profile, +Tachyon collects approximately 100,000 samples. If a function appears in 5,000 +samples (5% of total), Tachyon estimates it consumed 5% of the 10-second +duration, or about 500 milliseconds. This is a statistical estimate, not a +precise measurement. + +The accuracy of these estimates depends on sample count. With 100,000 samples, +a function showing 5% has a margin of error of roughly ±0.5%. With only 1,000 +samples, the same 5% measurement could actually represent anywhere from 3% to +7% of real time. + +This is why longer profiling durations and shorter sampling intervals produce +more reliable results---they collect more samples. For most performance +analysis, the default settings provide sufficient accuracy to identify +bottlenecks and guide optimization efforts. + +Because sampling is statistical, results will vary slightly between runs. A +function showing 12% in one run might show 11% or 13% in the next. This is +normal and expected. Focus on the overall pattern rather than exact percentages, +and don't worry about small variations between runs. + + +When to use a different approach +-------------------------------- + +Statistical sampling is not ideal for every situation. + +For very short scripts that complete in under one second, the profiler may not +collect enough samples for reliable results. Use :mod:`profiling.tracing` +instead, or run the script in a loop to extend profiling time. + +When you need exact call counts, sampling cannot provide them. Sampling +estimates frequency from snapshots, so if you need to know precisely how many +times a function was called, use :mod:`profiling.tracing`. + +When comparing two implementations where the difference might be only 1-2%, +sampling noise can obscure real differences. Use :mod:`timeit` for +micro-benchmarks or :mod:`profiling.tracing` for precise measurements. + + +The key difference from :mod:`profiling.tracing` is how measurement happens. +A tracing profiler instruments your code, recording every function call and +return. This provides exact call counts and precise timing but adds overhead +to every function call. A sampling profiler, by contrast, observes the program +from outside at fixed intervals without modifying its execution. Think of the +difference like this: tracing is like having someone follow you and write down +every step you take, while sampling is like taking photographs every second +and inferring your path from those snapshots. + +This external observation model is what makes sampling profiling practical for +production use. The profiled program runs at full speed because there is no +instrumentation code running inside it, and the target process is never stopped +or paused during sampling---Tachyon reads the call stack directly from the +process's memory while it continues to run. You can attach to a live server, +collect data, and detach without the application ever knowing it was observed. +The trade-off is that very short-lived functions may be missed if they happen +to complete between samples. + +Statistical profiling excels at answering the question, "Where is my program +spending time?" It reveals hotspots and bottlenecks in production code where +deterministic profiling overhead would be unacceptable. For exact call counts +and complete call graphs, use :mod:`profiling.tracing` instead. + + +Quick examples +============== + +Profile a script and see the results immediately:: + + python -m profiling.sampling run script.py + +Profile a module with arguments:: + + python -m profiling.sampling run -m mypackage.module arg1 arg2 + +Generate an interactive flame graph:: + + python -m profiling.sampling run --flamegraph -o profile.html script.py + +Attach to a running process by PID:: + + python -m profiling.sampling attach 12345 + +Use live mode for real-time monitoring (press ``q`` to quit):: + + python -m profiling.sampling run --live script.py + +Profile for 60 seconds with a faster sampling rate:: + + python -m profiling.sampling run -d 60 -r 20khz script.py + +Generate a line-by-line heatmap:: + + python -m profiling.sampling run --heatmap script.py + +Enable opcode-level profiling to see which bytecode instructions are executing:: + + python -m profiling.sampling run --opcodes --flamegraph script.py + + +Commands +======== + +Tachyon operates through two subcommands that determine how to obtain the +target process. + + +The ``run`` command +------------------- + +The ``run`` command launches a Python script or module and profiles it from +startup:: + + python -m profiling.sampling run script.py + python -m profiling.sampling run -m mypackage.module + +When profiling a script, the profiler starts the target in a subprocess, waits +for it to initialize, then begins collecting samples. The ``-m`` flag +indicates that the target should be run as a module (equivalent to +``python -m``). Arguments after the target are passed through to the +profiled program:: + + python -m profiling.sampling run script.py --config settings.yaml + + +The ``attach`` command +---------------------- + +The ``attach`` command connects to an already-running Python process by its +process ID:: + + python -m profiling.sampling attach 12345 + +This command is particularly valuable for investigating performance issues in +production systems. The target process requires no modification and need not +be restarted. The profiler attaches, collects samples for the specified +duration, then detaches and produces output. + +:: + + python -m profiling.sampling attach --live 12345 + python -m profiling.sampling attach --flamegraph -d 30 -o profile.html 12345 + +On most systems, attaching to another process requires appropriate permissions. +See :ref:`profiling-permissions` for platform-specific requirements. + + +.. _replay-command: + +The ``replay`` command +---------------------- + +The ``replay`` command converts binary profile files to other output formats:: + + python -m profiling.sampling replay profile.bin + python -m profiling.sampling replay --flamegraph -o profile.html profile.bin + +This command is useful when you have captured profiling data in binary format +and want to analyze it later or convert it to a visualization format. Binary +profiles can be replayed multiple times to different formats without +re-profiling. + +:: + + # Convert binary to pstats (default, prints to stdout) + python -m profiling.sampling replay profile.bin + + # Convert binary to flame graph + python -m profiling.sampling replay --flamegraph -o output.html profile.bin + + # Convert binary to gecko format for Firefox Profiler + python -m profiling.sampling replay --gecko -o profile.json profile.bin + + # Convert binary to heatmap + python -m profiling.sampling replay --heatmap -o my_heatmap profile.bin + + +Profiling in production +----------------------- + +The sampling profiler is designed for production use. It imposes no measurable +overhead on the target process because it reads memory externally rather than +instrumenting code. The target application continues running at full speed and +is unaware it is being profiled. + +When profiling production systems, keep these guidelines in mind: + +Start with shorter durations (10-30 seconds) to get quick results, then extend +if you need more statistical accuracy. By default, profiling runs until the +target process completes, which is usually sufficient to identify major hotspots. + +If possible, profile during representative load rather than peak traffic. +Profiles collected during normal operation are easier to interpret than those +collected during unusual spikes. + +The profiler itself consumes some CPU on the machine where it runs (not on the +target process). On the same machine, this is typically negligible. When +profiling remote processes, network latency does not affect the target. + +Results from production may differ from development due to different data +sizes, concurrent load, or caching effects. This is expected and is often +exactly what you want to capture. + + +.. _profiling-permissions: + +Platform requirements +--------------------- + +The profiler reads the target process's memory to capture stack traces. This +requires elevated permissions on most operating systems. + +**Linux** + +On Linux, the profiler uses ``ptrace`` or ``process_vm_readv`` to read the +target process's memory. This typically requires one of: + +- Running as root +- Having the ``CAP_SYS_PTRACE`` capability +- Adjusting the Yama ptrace scope: ``/proc/sys/kernel/yama/ptrace_scope`` + +The default ptrace_scope of 1 restricts ptrace to parent processes only. To +allow attaching to any process owned by the same user, set it to 0:: + + echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope + +**macOS** + +On macOS, the profiler uses ``task_for_pid()`` to access the target process. +This requires one of: + +- Running as root +- The profiler binary having the ``com.apple.security.cs.debugger`` entitlement +- System Integrity Protection (SIP) being disabled (not recommended) + +**Windows** + +On Windows, the profiler requires administrative privileges or the +``SeDebugPrivilege`` privilege to read another process's memory. + + +Version compatibility +--------------------- + +The profiler and target process must run the same Python minor version (for +example, both Python 3.15). Attaching from Python 3.14 to a Python 3.15 process +is not supported. + +Additional restrictions apply to pre-release Python versions: if either the +profiler or target is running a pre-release (alpha, beta, or release candidate), +both must run the exact same version. + +On free-threaded Python builds, the profiler cannot attach from a free-threaded +build to a standard build, or vice versa. + + +Sampling configuration +====================== + +Before exploring the various output formats and visualization options, it is +important to understand how to configure the sampling process itself. The +profiler offers several options that control how frequently samples are +collected, how long profiling runs, which threads are observed, and what +additional context is captured in each sample. + +The default configuration works well for most use cases: + +.. list-table:: + :header-rows: 1 + :widths: 25 75 + + * - Option + - Default + * - Default for ``--sampling-rate`` / ``-r`` + - 1 kHz + * - Default for ``--duration`` / ``-d`` + - Run to completion + * - Default for ``--all-threads`` / ``-a`` + - Main thread only + * - Default for ``--native`` + - No ```` frames (C code time attributed to caller) + * - Default for ``--no-gc`` + - ```` frames included when garbage collection is active + * - Default for ``--mode`` + - Wall-clock mode (all samples recorded) + * - Default for ``--realtime-stats`` + - Disabled + * - Default for ``--subprocesses`` + - Disabled + * - Default for ``--blocking`` + - Disabled (non-blocking sampling) + + +Sampling rate and duration +-------------------------- + +The two most fundamental parameters are the sampling rate and duration. +Together, these determine how many samples will be collected during a profiling +session. + +The :option:`--sampling-rate` option (:option:`-r`) sets how frequently samples +are collected. The default is 1 kHz (1,000 samples per second):: + + python -m profiling.sampling run -r 20khz script.py + +Higher rates capture more samples and provide finer-grained data at the +cost of slightly higher profiler CPU usage. Lower rates reduce profiler +overhead but may miss short-lived functions. For most applications, the +default rate provides a good balance between accuracy and overhead. + +The :option:`--duration` option (:option:`-d`) sets how long to profile in seconds. By +default, profiling continues until the target process exits or is interrupted:: + + python -m profiling.sampling run -d 60 script.py + +Specifying a duration is useful when attaching to long-running processes or when +you want to limit profiling to a specific time window. When profiling a script, +the default behavior of running to completion is usually what you want. + + +Thread selection +---------------- + +Python programs often use multiple threads, whether explicitly through the +:mod:`threading` module or implicitly through libraries that manage thread +pools. + +By default, the profiler samples only the main thread. The :option:`--all-threads` +option (:option:`-a`) enables sampling of all threads in the process:: + + python -m profiling.sampling run -a script.py + +Multi-thread profiling reveals how work is distributed across threads and can +identify threads that are blocked or starved. Each thread's samples are +combined in the output, with the ability to filter by thread in some formats. +This option is particularly useful when investigating concurrency issues or +when work is distributed across a thread pool. + + +.. _blocking-mode: + +Blocking mode +------------- + +By default, Tachyon reads the target process's memory without stopping it. +This non-blocking approach is ideal for most profiling scenarios because it +imposes virtually zero overhead on the target application: the profiled +program runs at full speed and is unaware it is being observed. + +However, non-blocking sampling can occasionally produce incomplete or +inconsistent stack traces in applications with many generators or coroutines +that rapidly switch between yield points, or in programs with very fast-changing +call stacks where functions enter and exit between the start and end of a single +stack read, resulting in reconstructed stacks that mix frames from different +execution states or that never actually existed. + +For these cases, the :option:`--blocking` option stops the target process during +each sample:: + + python -m profiling.sampling run --blocking script.py + python -m profiling.sampling attach --blocking 12345 + +When blocking mode is enabled, the profiler suspends the target process, +reads its stack, then resumes it. This guarantees that each captured stack +represents a real, consistent snapshot of what the process was doing at that +instant. The trade-off is that the target process runs slower because it is +repeatedly paused. + +.. warning:: + + Do not use very high sample rates (low ``--interval`` values) with blocking + mode. Suspending and resuming a process takes time, and if the sampling + interval is too short, the target will spend more time stopped than running. + For blocking mode, intervals of 1000 microseconds (1 millisecond) or higher + are recommended. The default 100 microsecond interval may cause noticeable + slowdown in the target application. + +Use blocking mode only when you observe inconsistent stacks in your profiles, +particularly with generator-heavy or coroutine-heavy code. For most +applications, the default non-blocking mode provides accurate results with +zero impact on the target process. + + +Special frames +-------------- + +The profiler can inject artificial frames into the captured stacks to provide +additional context about what the interpreter is doing at the moment each +sample is taken. These synthetic frames help distinguish different types of +execution that would otherwise be invisible. + +The :option:`--native` option adds ```` frames to indicate when Python has +called into C code (extension modules, built-in functions, or the interpreter +itself):: + + python -m profiling.sampling run --native script.py + +These frames help distinguish time spent in Python code versus time spent in +native libraries. Without this option, native code execution appears as time +in the Python function that made the call. This is useful when optimizing +code that makes heavy use of C extensions like NumPy or database drivers. + +By default, the profiler includes ```` frames when garbage collection is +active. The :option:`--no-gc` option suppresses these frames:: + + python -m profiling.sampling run --no-gc script.py + +GC frames help identify programs where garbage collection consumes significant +time, which may indicate memory allocation patterns worth optimizing. If you +see substantial time in ```` frames, consider investigating object +allocation rates or using object pooling. + + +Opcode-aware profiling +---------------------- + +The :option:`--opcodes` option enables instruction-level profiling that captures +which Python bytecode instructions are executing at each sample:: + + python -m profiling.sampling run --opcodes --flamegraph script.py + +This feature provides visibility into Python's bytecode execution, including +adaptive specialization optimizations. When a generic instruction like +``LOAD_ATTR`` is specialized at runtime into a more efficient variant like +``LOAD_ATTR_INSTANCE_VALUE``, the profiler shows both the specialized name +and the base instruction. + +Opcode information appears in several output formats: + +- **Flame graphs**: Hovering over a frame displays a tooltip with a bytecode + instruction breakdown, showing which opcodes consumed time in that function +- **Heatmap**: Expandable bytecode panels per source line show instruction + breakdown with specialization percentages +- **Live mode**: An opcode panel shows instruction-level statistics for the + selected function, accessible via keyboard navigation +- **Gecko format**: Opcode transitions are emitted as interval markers in the + Firefox Profiler timeline + +This level of detail is particularly useful for: + +- Understanding the performance impact of Python's adaptive specialization +- Identifying hot bytecode instructions that might benefit from optimization +- Analyzing the effectiveness of different code patterns at the instruction level +- Debugging performance issues that occur at the bytecode level + +The :option:`--opcodes` option is compatible with :option:`--live`, :option:`--flamegraph`, +:option:`--heatmap`, and :option:`--gecko` formats. It requires additional memory to store +opcode information and may slightly reduce sampling performance, but provides +unprecedented visibility into Python's execution model. + + +Real-time statistics +-------------------- + +The :option:`--realtime-stats` option displays sampling rate statistics during +profiling:: + + python -m profiling.sampling run --realtime-stats script.py + +This shows the actual achieved sampling rate, which may be lower than requested +if the profiler cannot keep up. The statistics help verify that profiling is +working correctly and that sufficient samples are being collected. See +:ref:`sampling-efficiency` for details on interpreting these metrics. + + +Subprocess profiling +-------------------- + +The :option:`--subprocesses` option enables automatic profiling of subprocesses +spawned by the target:: + + python -m profiling.sampling run --subprocesses script.py + python -m profiling.sampling attach --subprocesses 12345 + +When enabled, the profiler monitors the target process for child process +creation. When a new Python child process is detected, a separate profiler +instance is automatically spawned to profile it. This is useful for +applications that use :mod:`multiprocessing`, :mod:`subprocess`, +:mod:`concurrent.futures` with :class:`~concurrent.futures.ProcessPoolExecutor`, +or other process spawning mechanisms. + +.. code-block:: python + :caption: worker_pool.py + + from concurrent.futures import ProcessPoolExecutor + import math + + def compute_factorial(n): + total = 0 + for i in range(50): + total += math.factorial(n) + return total + + if __name__ == "__main__": + numbers = [5000 + i * 100 for i in range(50)] + with ProcessPoolExecutor(max_workers=4) as executor: + results = list(executor.map(compute_factorial, numbers)) + print(f"Computed {len(results)} factorials") + +:: + + python -m profiling.sampling run --subprocesses --flamegraph worker_pool.py + +This produces separate flame graphs for the main process and each worker +process: ``flamegraph_.html``, ``flamegraph_.html``, +and so on. + +Each subprocess receives its own output file. The filename is derived from +the specified output path (or the default) with the subprocess's process ID +appended: + +- If you specify ``-o profile.html``, subprocesses produce ``profile_12345.html``, + ``profile_12346.html``, and so on +- With default output, subprocesses produce files like ``flamegraph_12345.html`` + or directories like ``heatmap_12345`` +- For pstats format (which defaults to stdout), subprocesses produce files like + ``profile_12345.pstats`` + +The subprocess profilers inherit most sampling options from the parent (sampling +rate, duration, thread selection, native frames, GC frames, async-aware mode, +and output format). All Python descendant processes are profiled recursively, +including grandchildren and further descendants. + +Subprocess detection works by periodically scanning for new descendants of +the target process and checking whether each new process is a Python process +by probing the process memory for Python runtime structures. Non-Python +subprocesses (such as shell commands or external tools) are ignored. + +There is a limit of 100 concurrent subprocess profilers to prevent resource +exhaustion in programs that spawn many processes. If this limit is reached, +additional subprocesses are not profiled and a warning is printed. + +The :option:`--subprocesses` option is incompatible with :option:`--live` mode +because live mode uses an interactive terminal interface that cannot +accommodate multiple concurrent profiler displays. + + +.. _sampling-efficiency: + +Sampling efficiency +------------------- + +Sampling efficiency metrics help assess the quality of the collected data. +These metrics appear in the profiler's terminal output and in the flame graph +sidebar. + +**Sampling efficiency** is the percentage of sample attempts that succeeded. +Each sample attempt reads the target process's call stack from memory. An +attempt can fail if the process is in an inconsistent state at the moment of +reading, such as during a context switch or while the interpreter is updating +its internal structures. A low efficiency may indicate that the profiler could +not keep up with the requested sampling rate, often due to system load or an +overly aggressive interval setting. + +**Missed samples** is the percentage of expected samples that were not +collected. Based on the configured interval and duration, the profiler expects +to collect a certain number of samples. Some samples may be missed if the +profiler falls behind schedule, for example when the system is under heavy +load. A small percentage of missed samples is normal and does not significantly +affect the statistical accuracy of the profile. + +Both metrics are informational. Even with some failed attempts or missed +samples, the profile remains statistically valid as long as enough samples +were collected. The profiler reports the actual number of samples captured, +which you can use to judge whether the data is sufficient for your analysis. + + +Profiling modes +=============== + +The sampling profiler supports four modes that control which samples are +recorded. The mode determines what the profile measures: total elapsed time, +CPU execution time, time spent holding the global interpreter lock, or +exception handling. + + +Wall-clock mode +--------------- + +Wall-clock mode (:option:`--mode`\ ``=wall``) captures all samples regardless of what the +thread is doing. This is the default mode and provides a complete picture of +where time passes during program execution:: + + python -m profiling.sampling run --mode=wall script.py + +In wall-clock mode, samples are recorded whether the thread is actively +executing Python code, waiting for I/O, blocked on a lock, or sleeping. +This makes wall-clock profiling ideal for understanding the overall time +distribution in your program, including time spent waiting. + +If your program spends significant time in I/O operations, network calls, or +sleep, wall-clock mode will show these waits as time attributed to the calling +function. This is often exactly what you want when optimizing end-to-end +latency. + + +CPU mode +-------- + +CPU mode (:option:`--mode`\ ``=cpu``) records samples only when the thread is actually +executing on a CPU core:: + + python -m profiling.sampling run --mode=cpu script.py + +Samples taken while the thread is sleeping, blocked on I/O, or waiting for +a lock are discarded. The resulting profile shows where CPU cycles are consumed, +filtering out idle time. + +CPU mode is useful when you want to focus on computational hotspots without +being distracted by I/O waits. If your program alternates between computation +and network calls, CPU mode reveals which computational sections are most +expensive. + + +Comparing wall-clock and CPU profiles +------------------------------------- + +Running both wall-clock and CPU mode profiles can reveal whether a function's +time is spent computing or waiting. + +If a function appears prominently in both profiles, it is a true computational +hotspot---actively using the CPU. Optimization should focus on algorithmic +improvements or more efficient code. + +If a function is high in wall-clock mode but low or absent in CPU mode, it is +I/O-bound or waiting. The function spends most of its time waiting for network, +disk, locks, or sleep. CPU optimization won't help here; consider async I/O, +connection pooling, or reducing wait time instead. + +.. code-block:: python + + import time + + def do_sleep(): + time.sleep(2) + + def do_compute(): + sum(i**2 for i in range(1000000)) + + if __name__ == "__main__": + do_sleep() + do_compute() + +:: + + python -m profiling.sampling run --mode=wall script.py # do_sleep ~98%, do_compute ~1% + python -m profiling.sampling run --mode=cpu script.py # do_sleep absent, do_compute dominates + + +GIL mode +-------- + +GIL mode (:option:`--mode`\ ``=gil``) records samples only when the thread holds Python's +global interpreter lock:: + + python -m profiling.sampling run --mode=gil script.py + +The GIL is held only while executing Python bytecode. When Python calls into +C extensions, performs I/O operations, or executes native code, the GIL is +typically released. This means GIL mode effectively measures time spent +running Python code specifically, filtering out time in native libraries. + +In multi-threaded programs, GIL mode reveals which code is preventing other +threads from running Python bytecode. Since only one thread can hold the GIL +at a time, functions that appear frequently in GIL mode profiles are +monopolizing the interpreter. + +GIL mode helps answer questions like "which functions are monopolizing the +GIL?" and "why are my other threads starving?" It can also be useful in +single-threaded programs to distinguish Python execution time from time spent +in C extensions or I/O. + +.. code-block:: python + + import hashlib + + def hash_work(): + # C extension - releases GIL during computation + for _ in range(200): + hashlib.sha256(b"data" * 250000).hexdigest() + + def python_work(): + # Pure Python - holds GIL during computation + for _ in range(3): + sum(i**2 for i in range(1000000)) + + if __name__ == "__main__": + hash_work() + python_work() + +:: + + python -m profiling.sampling run --mode=cpu script.py # hash_work ~42%, python_work ~38% + python -m profiling.sampling run --mode=gil script.py # hash_work ~5%, python_work ~60% + + +Exception mode +-------------- + +Exception mode (``--mode=exception``) records samples only when a thread has +an active exception:: + + python -m profiling.sampling run --mode=exception script.py + +Samples are recorded in two situations: when an exception is being propagated +up the call stack (after ``raise`` but before being caught), or when code is +executing inside an ``except`` block where exception information is still +present in the thread state. + +The following example illustrates which code regions are captured: + +.. code-block:: python + + def example(): + try: + raise ValueError("error") # Captured: exception being raised + except ValueError: + process_error() # Captured: inside except block + finally: + cleanup() # NOT captured: exception already handled + + def example_propagating(): + try: + try: + raise ValueError("error") + finally: + cleanup() # Captured: exception propagating through + except ValueError: + pass + + def example_no_exception(): + try: + do_work() + finally: + cleanup() # NOT captured: no exception involved + +Note that ``finally`` blocks are only captured when an exception is actively +propagating through them. Once an ``except`` block finishes executing, Python +clears the exception information before running any subsequent ``finally`` +block. Similarly, ``finally`` blocks that run during normal execution (when no +exception was raised) are not captured because no exception state is present. + +This mode is useful for understanding where your program spends time handling +errors. Exception handling can be a significant source of overhead in code +that uses exceptions for flow control (such as ``StopIteration`` in iterators) +or in applications that process many error conditions (such as network servers +handling connection failures). + +Exception mode helps answer questions like "how much time is spent handling +exceptions?" and "which exception handlers are the most expensive?" It can +reveal hidden performance costs in code that catches and processes many +exceptions, even when those exceptions are handled gracefully. For example, +if a parsing library uses exceptions internally to signal format errors, this +mode will capture time spent in those handlers even if the calling code never +sees the exceptions. + + +Output formats +============== + +The profiler produces output in several formats, each suited to different +analysis workflows. The format is selected with a command-line flag, and +output goes to stdout, a file, or a directory depending on the format. + + +pstats format +------------- + +The pstats format (:option:`--pstats`) produces a text table similar to what +deterministic profilers generate. This is the default output format:: + + python -m profiling.sampling run script.py + python -m profiling.sampling run --pstats script.py + +.. figure:: tachyon-pstats.png + :alt: Tachyon pstats terminal output + :align: center + :width: 100% + + The pstats format displays profiling results in a color-coded table showing + function hotspots, sample counts, and timing estimates. + +Output appears on stdout by default:: + + Profile Stats (Mode: wall): + nsamples sample% tottime (ms) cumul% cumtime (ms) filename:lineno(function) + 234/892 11.7% 234.00 44.6% 892.00 server.py:145(handle_request) + 156/156 7.8% 156.00 7.8% 156.00 :0(socket.recv) + 98/421 4.9% 98.00 21.1% 421.00 parser.py:67(parse_message) + +The columns show sampling counts and estimated times: + +- **nsamples**: Displayed as ``direct/cumulative`` (for example, ``10/50``). + Direct samples are when the function was at the top of the stack, actively + executing. Cumulative samples are when the function appeared anywhere on the + stack, including when it was waiting for functions it called. If a function + shows ``10/50``, it was directly executing in 10 samples and was on the call + stack in 50 samples total. + +- **sample%** and **cumul%**: Percentages of total samples for direct and + cumulative counts respectively. + +- **tottime** and **cumtime**: Estimated wall-clock time based on sample counts + and the profiling duration. Time units are selected automatically based on + the magnitude: seconds for large values, milliseconds for moderate values, + or microseconds for small values. + +The output includes a legend explaining each column and a summary of +interesting functions that highlights: + +- **Hot spots**: Functions with high direct/cumulative sample ratio (ratio + close to 1.0). These functions spend most of their time executing their own + code rather than waiting for callees. High ratios indicate where CPU time + is actually consumed. + +- **Indirect calls**: Functions with large differences between cumulative and + direct samples. These are orchestration functions that delegate work to + other functions. They appear frequently on the stack but rarely at the top. + +- **Call magnification**: Functions where cumulative samples far exceed direct + samples (high cumulative/direct multiplier). These are frequently-nested + functions that appear deep in many call chains. + +Use :option:`--no-summary` to suppress both the legend and summary sections. + +To save pstats output to a binary file instead of stdout:: + + python -m profiling.sampling run -o profile.pstats script.py + +The pstats format supports several options for controlling the display. +The :option:`--sort` option determines the column used for ordering results:: + + python -m profiling.sampling run --sort=tottime script.py + python -m profiling.sampling run --sort=cumtime script.py + python -m profiling.sampling run --sort=nsamples script.py + +The :option:`--limit` option restricts output to the top N entries:: + + python -m profiling.sampling run --limit=30 script.py + +The :option:`--no-summary` option suppresses the header summary that precedes the +statistics table. + + +Collapsed stacks format +----------------------- + +Collapsed stacks format (:option:`--collapsed`) produces one line per unique call +stack, with a count of how many times that stack was sampled:: + + python -m profiling.sampling run --collapsed script.py + +The output looks like: + +.. code-block:: text + + main;process_data;parse_json;decode_utf8 42 + main;process_data;parse_json 156 + main;handle_request;send_response 89 + +Each line contains semicolon-separated function names representing the call +stack from bottom to top, followed by a space and the sample count. This +format is designed for compatibility with external flame graph tools, +particularly Brendan Gregg's ``flamegraph.pl`` script. + +To generate a flame graph from collapsed stacks:: + + python -m profiling.sampling run --collapsed script.py > stacks.txt + flamegraph.pl stacks.txt > profile.svg + +The resulting SVG can be viewed in any web browser and provides an interactive +visualization where you can click to zoom into specific call paths. + + +Flame graph format +------------------ + +Flame graph format (:option:`--flamegraph`) produces a self-contained HTML file with +an interactive flame graph visualization:: + + python -m profiling.sampling run --flamegraph script.py + python -m profiling.sampling run --flamegraph -o profile.html script.py + +.. figure:: tachyon-flamegraph.png + :alt: Tachyon interactive flame graph + :align: center + :width: 100% + + The flame graph visualization shows call stacks as nested rectangles, with + width proportional to time spent. The sidebar displays runtime statistics, + GIL metrics, and hotspot functions. + +.. only:: html + + `Try the interactive example <../_static/tachyon-example-flamegraph.html>`__! + +If no output file is specified, the profiler generates a filename based on +the process ID (for example, ``flamegraph.12345.html``). + +The generated HTML file requires no external dependencies and can be opened +directly in a web browser. The visualization displays call stacks as nested +rectangles, with width proportional to time spent. Hovering over a rectangle +shows details about that function including source code context, and clicking +zooms into that portion of the call tree. + +The flame graph interface includes: + +- A sidebar showing profile summary, thread statistics, sampling efficiency + metrics (see :ref:`sampling-efficiency`), and top hotspot functions +- Search functionality supporting both function name matching and + ``file.py:42`` line patterns +- Per-thread filtering via dropdown +- Dark/light theme toggle (preference saved across sessions) +- SVG export for saving the current view + +The thread statistics section shows runtime behavior metrics: + +- **GIL Held**: percentage of samples where a thread held the global interpreter + lock (actively running Python code) +- **GIL Released**: percentage of samples where no thread held the GIL +- **Waiting GIL**: percentage of samples where a thread was waiting to acquire + the GIL +- **GC**: percentage of samples during garbage collection + +These statistics help identify GIL contention and understand how time is +distributed between Python execution, native code, and waiting. + +Flame graphs are particularly effective for identifying deep call stacks and +understanding the hierarchical structure of time consumption. Wide rectangles +at the top indicate functions that consume significant time either directly +or through their callees. + + +Differential flame graphs +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Differential flame graphs compare two profiling runs to highlight where +performance changed. This helps identify regressions introduced by code changes +and validate that optimizations achieved their intended effect:: + + # Capture baseline profile + python -m profiling.sampling run --binary -o baseline.bin script.py + + # After modifying code, generate differential flamegraph + python -m profiling.sampling run --diff-flamegraph baseline.bin -o diff.html script.py + +The visualization draws the current profile with frame widths showing current +time consumption, then applies color to indicate how each function changed +relative to the baseline. + +**Color coding**: + +- **Red**: Functions consuming more time (regressions). Lighter shades indicate + modest increases, while darker shades show severe regressions. + +- **Blue**: Functions consuming less time (improvements). Lighter shades for + modest reductions, darker shades for significant speedups. + +- **Gray**: Minimal or no change. + +- **Purple**: New functions not present in the baseline. + +Frame colors indicate changes in **direct time** (time when the function was at +the top of the stack, actively executing), not cumulative time including callees. +Hovering over a frame shows comparison details including baseline time, current +time, and the percentage change. + +Some call paths may disappear entirely between profiles. These are called +**elided stacks** and occur when optimizations eliminate code paths or certain +branches stop executing. If elided stacks are present, an elided toggle appears +allowing you to switch between the main differential view and an elided-only +view that shows just the removed paths (colored purple). + + +Gecko format +------------ + +Gecko format (:option:`--gecko`) produces JSON output compatible with the Firefox +Profiler:: + + python -m profiling.sampling run --gecko script.py + python -m profiling.sampling run --gecko -o profile.json script.py + +The `Firefox Profiler `__ is a sophisticated +web-based tool originally built for profiling Firefox itself. It provides +features beyond basic flame graphs, including a timeline view, call tree +exploration, and marker visualization. See the +`Firefox Profiler documentation `__ for +detailed usage instructions. + +To use the output, open the Firefox Profiler in your browser and load the +JSON file. The profiler runs entirely client-side, so your profiling data +never leaves your machine. + +Gecko format automatically collects additional metadata about GIL state and +CPU activity, enabling analysis features specific to Python's threading model. +The profiler emits interval markers that appear as colored bands in the +Firefox Profiler timeline: + +- **GIL markers**: show when threads hold or release the global interpreter lock +- **CPU markers**: show when threads are executing on CPU versus idle +- **Code type markers**: distinguish Python code from native (C extension) code +- **GC markers**: indicate garbage collection activity + +For this reason, the :option:`--mode` option is not available with Gecko format; +all relevant data is captured automatically. + +.. figure:: tachyon-gecko-calltree.png + :alt: Firefox Profiler Call Tree view + :align: center + :width: 100% + + The Call Tree view shows the complete call hierarchy with sample counts + and percentages. The sidebar displays detailed statistics for the + selected function including running time and sample distribution. + +.. figure:: tachyon-gecko-flamegraph.png + :alt: Firefox Profiler Flame Graph view + :align: center + :width: 100% + + The Flame Graph visualization shows call stacks as nested rectangles. + Functions names are visible in the call hierarchy. + +.. figure:: tachyon-gecko-opcodes.png + :alt: Firefox Profiler Marker Chart with opcodes + :align: center + :width: 100% + + The Marker Chart displays interval markers including CPU state, GIL + status, and opcodes. With ``--opcodes`` enabled, bytecode instructions + like ``BINARY_OP_ADD_FLOAT``, ``CALL_PY_EXACT_ARGS``, and + ``CALL_LIST_APPEND`` appear as markers showing execution over time. + + +Heatmap format +-------------- + +Heatmap format (:option:`--heatmap`) generates an interactive HTML visualization +showing sample counts at the source line level:: + + python -m profiling.sampling run --heatmap script.py + python -m profiling.sampling run --heatmap -o my_heatmap script.py + +.. figure:: tachyon-heatmap.png + :alt: Tachyon heatmap visualization + :align: center + :width: 100% + + The heatmap overlays sample counts directly on your source code. Lines are + color-coded from cool (few samples) to hot (many samples). Navigation + buttons (▲▼) let you jump between callers and callees. + +Unlike other formats that produce a single file, heatmap output creates a +directory containing HTML files for each profiled source file. If no output +path is specified, the directory is named ``heatmap_PID``. + +The heatmap visualization displays your source code with a color gradient +indicating how many samples were collected at each line. Hot lines (many +samples) appear in warm colors, while cold lines (few or no samples) appear +in cool colors. This view helps pinpoint exactly which lines of code are +responsible for time consumption. + +The heatmap interface provides several interactive features: + +- **Coloring modes**: toggle between "Self Time" (direct execution) and + "Total Time" (cumulative, including time in called functions) +- **Cold code filtering**: show all lines or only lines with samples +- **Call graph navigation**: each line shows navigation buttons (▲ for callers, + ▼ for callees) that let you trace execution paths through your code. When + multiple functions called or were called from a line, a menu appears showing + all options with their sample counts. +- **Scroll minimap**: a vertical overview showing the heat distribution across + the entire file +- **Hierarchical index**: files organized by type (stdlib, site-packages, + project) with aggregate sample counts per folder +- **Dark/light theme**: toggle with preference saved across sessions +- **Line linking**: click line numbers to create shareable URLs + +When opcode-level profiling is enabled with :option:`--opcodes`, each hot line +can be expanded to show which bytecode instructions consumed time: + +.. figure:: tachyon-heatmap-with-opcodes.png + :alt: Heatmap with expanded bytecode panel + :align: center + :width: 100% + + Expanding a hot line reveals the bytecode instructions executed, including + specialized variants. The panel shows sample counts per instruction and the + overall specialization percentage for the line. + +.. only:: html + + `Try the interactive example <../_static/tachyon-example-heatmap.html>`__! + +Heatmaps are especially useful when you know which file contains a performance +issue but need to identify the specific lines. Many developers prefer this +format because it maps directly to their source code, making it easy to read +and navigate. For smaller scripts and focused analysis, heatmaps provide an +intuitive view that shows exactly where time is spent without requiring +interpretation of hierarchical visualizations. + + +Binary format +------------- + +Binary format (:option:`--binary`) produces a compact binary file for efficient +storage of profiling data:: + + python -m profiling.sampling run --binary -o profile.bin script.py + python -m profiling.sampling attach --binary -o profile.bin 12345 + +The :option:`--compression` option controls data compression: + +- ``auto`` (default): Use zstd compression if available, otherwise no + compression +- ``zstd``: Force zstd compression (requires :mod:`compression.zstd` support) +- ``none``: Disable compression + +:: + + python -m profiling.sampling run --binary --compression=zstd -o profile.bin script.py + +To analyze binary profiles, use the :ref:`replay-command` to convert them to +other formats like flame graphs or pstats output. + + +Record and replay workflow +========================== + +The binary format combined with the replay command enables a record-and-replay +workflow that separates data capture from analysis. Rather than generating +visualizations during profiling, you capture raw data to a compact binary file +and convert it to different formats later. + +This approach has three main benefits: + +- Sampling runs faster because the work of building data structures for + visualization is deferred until replay. +- A single binary capture can be converted to multiple output formats + without re-profiling: pstats for a quick overview, flame graph for visual + exploration, heatmap for line-level detail. +- Binary files are compact and easy to share with colleagues who can convert + them to their preferred format. + +A typical workflow:: + + # Capture profile in production or during tests + python -m profiling.sampling attach --binary -o profile.bin 12345 + + # Later, analyze with different formats + python -m profiling.sampling replay profile.bin + python -m profiling.sampling replay --flamegraph -o profile.html profile.bin + python -m profiling.sampling replay --heatmap -o heatmap profile.bin + + +Live mode +========= + +Live mode (:option:`--live`) provides a terminal-based real-time view of profiling +data, similar to the ``top`` command for system processes:: + + python -m profiling.sampling run --live script.py + python -m profiling.sampling attach --live 12345 + +.. only:: not latex + + .. figure:: tachyon-live-mode-2.gif + :alt: Tachyon live mode showing all threads + :align: center + :width: 100% + + Live mode displays real-time profiling statistics, showing combined + data from multiple threads in a multi-threaded application. + +The display updates continuously as new samples arrive, showing the current +hottest functions. This mode requires the :mod:`curses` module, which is +available on Unix-like systems but not on Windows. The terminal must be at +least 60 columns wide and 12 lines tall; larger terminals display more columns. + +The header displays the top 3 hottest functions, sampling efficiency metrics, +and thread status statistics (GIL held percentage, CPU usage, GC time). The +main table shows function statistics with the currently sorted column indicated +by an arrow (▼). + +When :option:`--opcodes` is enabled, an additional opcode panel appears below the +main table, showing instruction-level statistics for the currently selected +function. This panel displays which bytecode instructions are executing most +frequently, including specialized variants and their base opcodes. + +.. only:: not latex + + .. figure:: tachyon-live-mode-1.gif + :alt: Tachyon live mode with opcode panel + :align: center + :width: 100% + + Live mode with ``--opcodes`` enabled shows an opcode panel with a bytecode + instruction breakdown for the selected function. + + +Keyboard commands +----------------- + +Within live mode, keyboard commands control the display: + +:kbd:`q` + Quit the profiler and return to the shell. + +:kbd:`s` / :kbd:`S` + Cycle through sort orders forward/backward (sample count, percentage, + total time, cumulative percentage, cumulative time). + +:kbd:`p` + Pause or resume display updates. Sampling continues in the background + while the display is paused, so you can freeze the view to examine results + without stopping data collection. + +:kbd:`r` + Reset all statistics and start fresh. This is disabled after profiling + finishes to prevent accidental data loss. + +:kbd:`/` + Enter filter mode to search for functions by name. The filter uses + case-insensitive substring matching against the filename and function name. + Type a pattern and press Enter to apply, or Escape to cancel. Glob patterns + and regular expressions are not supported. + +:kbd:`c` + Clear the current filter and show all functions again. + +:kbd:`t` + Toggle between viewing all threads combined or per-thread statistics. + In per-thread mode, a thread counter (for example, ``1/4``) appears showing + your position among the available threads. + +:kbd:`←` :kbd:`→` or :kbd:`↑` :kbd:`↓` + In per-thread view, navigate between threads. Navigation wraps around + from the last thread to the first and vice versa. + +:kbd:`+` / :kbd:`-` + Increase or decrease the display refresh rate. The range is 0.05 seconds + (20 Hz, very responsive) to 1.0 second (1 Hz, lower overhead). Faster refresh + rates use more CPU. The default is 0.1 seconds (10 Hz). + +:kbd:`x` + Toggle trend indicators that show whether functions are becoming hotter + or cooler over time. When enabled, increasing metrics appear in green and + decreasing metrics appear in red, comparing each update to the previous one. + +:kbd:`h` or :kbd:`?` + Show the help screen with all available commands. + +:kbd:`j` / :kbd:`k` (or :kbd:`Up` / :kbd:`Down`) + Navigate through opcode entries in the opcode panel (when ``--opcodes`` is + enabled). These keys scroll through the instruction-level statistics for the + currently selected function. + +When profiling finishes (duration expires or target process exits), the display +shows a "PROFILING COMPLETE" banner and freezes the final results. You can +still navigate, sort, and filter the results before pressing :kbd:`q` to exit. + +Live mode is incompatible with output format options (:option:`--collapsed`, +:option:`--flamegraph`, and so on) because it uses an interactive terminal +interface rather than producing file output. + + +Async-aware profiling +===================== + +For programs using :mod:`asyncio`, the profiler offers async-aware mode +(:option:`--async-aware`) that reconstructs call stacks based on the task structure +rather than the raw Python frames:: + + python -m profiling.sampling run --async-aware async_script.py + +Standard profiling of async code can be confusing because the physical call +stack often shows event loop internals rather than the logical flow of your +coroutines. Async-aware mode addresses this by tracking which task is running +and presenting stacks that reflect the ``await`` chain. + +.. code-block:: python + + import asyncio + + async def fetch(url): + await asyncio.sleep(0.1) + return url + + async def main(): + for _ in range(50): + await asyncio.gather(fetch("a"), fetch("b"), fetch("c")) + + if __name__ == "__main__": + asyncio.run(main()) + +:: + + python -m profiling.sampling run --async-aware --flamegraph -o out.html script.py + +.. note:: + + Async-aware profiling requires the target process to have the :mod:`asyncio` + module loaded. If you profile a script before it imports asyncio, async-aware + mode will not be able to capture task information. + + +Async modes +----------- + +The :option:`--async-mode` option controls which tasks appear in the profile:: + + python -m profiling.sampling run --async-aware --async-mode=running async_script.py + python -m profiling.sampling run --async-aware --async-mode=all async_script.py + +With :option:`--async-mode`\ ``=running`` (the default), only the task currently executing +on the CPU is profiled. This shows where your program is actively spending time +and is the typical choice for performance analysis. + +With :option:`--async-mode`\ ``=all``, tasks that are suspended (awaiting I/O, locks, or +other tasks) are also included. This mode is useful for understanding what your +program is waiting on, but produces larger profiles since every suspended task +appears in each sample. + + +Task markers and stack reconstruction +------------------------------------- + +In async-aware profiles, you will see ```` frames that mark boundaries +between asyncio tasks. These are synthetic frames inserted by the profiler to +show the task structure. The task name appears as the function name in these +frames. + +When a task awaits another task, the profiler reconstructs the logical call +chain by following the ``await`` relationships. Only "leaf" tasks (tasks that +no other task is currently awaiting) generate their own stack entries. Tasks +being awaited by other tasks appear as part of their awaiter's stack instead. + +If a task has multiple awaiters (a diamond pattern in the task graph), the +profiler deterministically selects one parent and annotates the task marker +with the number of parents, for example ``MyTask (2 parents)``. This indicates +that alternate execution paths exist but are not shown in this particular stack. + + +Option restrictions +------------------- + +Async-aware mode uses a different stack reconstruction mechanism and is +incompatible with: :option:`--native`, :option:`--no-gc`, :option:`--all-threads`, and +:option:`--mode`\ ``=cpu`` or :option:`--mode`\ ``=gil``. + + +Command-line interface +====================== + +.. program:: profiling.sampling + +The complete command-line interface for reference. + + +Global options +-------------- + +.. option:: run + + Run and profile a Python script or module. + +.. option:: attach + + Attach to and profile a running process by PID. + +.. option:: replay + + Convert a binary profile file to another output format. + + +Sampling options +---------------- + +.. option:: -r , --sampling-rate + + Sampling rate (for example, ``10000``, ``10khz``, ``10k``). Default: ``1khz``. + +.. option:: -d , --duration + + Profiling duration in seconds. Default: run to completion. + +.. option:: -a, --all-threads + + Sample all threads, not just the main thread. + +.. option:: --realtime-stats + + Display sampling statistics during profiling. + +.. option:: --native + + Include ```` frames for non-Python code. + +.. option:: --no-gc + + Exclude ```` frames for garbage collection. + +.. option:: --async-aware + + Enable async-aware profiling for asyncio programs. + +.. option:: --opcodes + + Gather bytecode opcode information for instruction-level profiling. Shows + which bytecode instructions are executing, including specializations. + Compatible with ``--live``, ``--flamegraph``, ``--heatmap``, and ``--gecko`` + formats only. + +.. option:: --subprocesses + + Also profile subprocesses. Each subprocess gets its own profiler + instance and output file. Incompatible with ``--live``. + +.. option:: --blocking + + Pause the target process during each sample. This ensures consistent + stack traces at the cost of slowing down the target. Use with longer + intervals (1000 µs or higher) to minimize impact. See :ref:`blocking-mode` + for details. + + +Mode options +------------ + +.. option:: --mode + + Sampling mode: ``wall`` (default), ``cpu``, ``gil``, or ``exception``. + The ``cpu``, ``gil``, and ``exception`` modes are incompatible with + ``--async-aware``. + +.. option:: --async-mode + + Async profiling mode: ``running`` (default) or ``all``. + Requires ``--async-aware``. + + +Output options +-------------- + +.. option:: --pstats + + Generate pstats statistics. This is the default. + When written to stdout, the output is a text table; with :option:`-o`, + it is a binary pstats file. + +.. option:: --collapsed + + Generate collapsed stack format for external flame graph tools. + +.. option:: --flamegraph + + Generate self-contained HTML flame graph. + +.. option:: --diff-flamegraph + + Generate differential flamegraph comparing to a baseline binary profile. + +.. option:: --gecko + + Generate Gecko JSON format for Firefox Profiler. + +.. option:: --heatmap + + Generate HTML heatmap with line-level sample counts. + +.. option:: --binary + + Generate high-performance binary format for later conversion with the + ``replay`` command. + +.. option:: --compression + + Compression for binary format: ``auto`` (use zstd if available, default), + ``zstd``, or ``none``. + +.. option:: -o , --output + + Output file or directory path. Default behavior varies by format: + :option:`--pstats` prints a text table to stdout, while ``-o`` writes a + binary pstats file. Other formats generate a file named + ``_.`` (for example, ``flamegraph_12345.html``). + :option:`--heatmap` creates a directory named ``heatmap_``. + +.. option:: --browser + + Automatically open HTML output (:option:`--flamegraph` and + :option:`--heatmap`) in your default web browser after generation. + When profiling with :option:`--subprocesses`, only the main process + opens the browser; subprocess outputs are never auto-opened. + + +pstats display options +---------------------- + +These options apply only to pstats format output. + +.. option:: --sort + + Sort order: ``nsamples``, ``tottime``, ``cumtime``, ``sample-pct``, + ``cumul-pct``, ``nsamples-cumul``, or ``name``. Default: ``nsamples``. + +.. option:: -l , --limit + + Maximum number of entries to display. Default: 15. + +.. option:: --no-summary + + Omit the Legend and Summary of Interesting Functions sections from output. + + +Run command options +------------------- + +.. option:: -m, --module + + Treat the target as a module name rather than a script path. + +.. option:: --live + + Start interactive terminal interface instead of batch profiling. + + +.. seealso:: + + :mod:`profiling` + Overview of Python profiling tools and guidance on choosing a profiler. + + :mod:`profiling.tracing` + Deterministic tracing profiler for exact call counts and timing. + + :mod:`pstats` + Statistics analysis for profile data. + + `Firefox Profiler `__ + Web-based profiler that accepts Gecko format output. See the + `documentation `__ for usage details. + + `FlameGraph `__ + Tools for generating flame graphs from collapsed stack format. diff --git a/Doc/library/profiling.tracing.rst b/Doc/library/profiling.tracing.rst new file mode 100644 index 00000000000..d45423cf0d8 --- /dev/null +++ b/Doc/library/profiling.tracing.rst @@ -0,0 +1,331 @@ +.. _profiling-tracing: + +**************************************************** +:mod:`!profiling.tracing` --- Deterministic profiler +**************************************************** + +.. module:: profiling.tracing + :synopsis: Deterministic tracing profiler for Python programs. + +.. module:: cProfile + :synopsis: Alias for profiling.tracing (backward compatibility). + :noindex: + +.. versionadded:: 3.15 + +**Source code:** :source:`Lib/profiling/tracing/` + +-------------- + +The :mod:`!profiling.tracing` module provides deterministic profiling of Python +programs. It monitors every function call, function return, and exception event, +recording precise timing for each. This approach provides exact call counts and +complete visibility into program execution, making it ideal for development and +testing scenarios. + +.. note:: + + This module is also available as ``cProfile`` for backward compatibility. + The ``cProfile`` name will continue to work in all future Python versions. + Use whichever import style suits your codebase:: + + # Preferred (new style) + import profiling.tracing + profiling.tracing.run('my_function()') + + # Also works (backward compatible) + import cProfile + cProfile.run('my_function()') + + +What is deterministic profiling? +================================ + +:dfn:`Deterministic profiling` captures every function call, function return, +and exception event during program execution. The profiler measures the precise +time intervals between these events, providing exact statistics about how the +program behaves. + +In contrast to :ref:`statistical profiling `, which samples +the call stack periodically to estimate where time is spent, deterministic +profiling records every event. This means you get exact call counts rather than +statistical approximations. The trade-off is that instrumenting every event +introduces overhead that can slow down program execution. + +Python's interpreted nature makes deterministic profiling practical. The +interpreter already dispatches events for function calls and returns, so the +profiler can hook into this mechanism without requiring code modification. The +overhead tends to be moderate relative to the inherent cost of interpretation, +making deterministic profiling suitable for most development workflows. + +Deterministic profiling helps answer questions like: + +- How many times was this function called? +- What is the complete call graph of my program? +- Which functions are called by a particular function? +- Are there unexpected function calls happening? + +Call count statistics can identify bugs (surprising counts) and inline +expansion opportunities (high call counts). Internal time statistics reveal +"hot loops" that warrant optimization. Cumulative time statistics help identify +algorithmic inefficiencies. The handling of cumulative times in this profiler +allows direct comparison of recursive and iterative implementations. + + +.. _profiling-tracing-cli: + +Command-line interface +====================== + +.. program:: profiling.tracing + +The :mod:`!profiling.tracing` module can be invoked as a script to profile +another script or module: + +.. code-block:: shell-session + + python -m profiling.tracing [-o output_file] [-s sort_order] (-m module | script.py) + +This runs the specified script or module under the profiler and prints the +results to standard output (or saves them to a file). + +.. option:: -o + + Write the profile results to a file instead of standard output. The output + file can be read by the :mod:`pstats` module for later analysis. + +.. option:: -s + + Sort the output by the specified key. This accepts any of the sort keys + recognized by :meth:`pstats.Stats.sort_stats`, such as ``cumulative``, + ``time``, ``calls``, or ``name``. This option only applies when + :option:`-o ` is not specified. + +.. option:: -m + + Profile a module instead of a script. The module is located using the + standard import mechanism. + + .. versionadded:: 3.7 + The ``-m`` option for ``cProfile``. + + .. versionadded:: 3.8 + The ``-m`` option for :mod:`profile`. + + +Programmatic usage examples +=========================== + +For more control over profiling, use the module's functions and classes +directly. + + +Basic profiling +--------------- + +The simplest approach uses the :func:`!run` function:: + + import profiling.tracing + profiling.tracing.run('my_function()') + +This profiles the given code string and prints a summary to standard output. +To save results for later analysis:: + + profiling.tracing.run('my_function()', 'output.prof') + + +Using the :class:`!Profile` class +--------------------------------- + +The :class:`!Profile` class provides fine-grained control:: + + import profiling.tracing + import pstats + from io import StringIO + + pr = profiling.tracing.Profile() + pr.enable() + # ... code to profile ... + pr.disable() + + # Print results + s = StringIO() + ps = pstats.Stats(pr, stream=s).sort_stats(pstats.SortKey.CUMULATIVE) + ps.print_stats() + print(s.getvalue()) + +The :class:`!Profile` class also works as a context manager:: + + import profiling.tracing + + with profiling.tracing.Profile() as pr: + # ... code to profile ... + + pr.print_stats() + + +Module reference +================ + +.. currentmodule:: profiling.tracing + +.. function:: run(command, filename=None, sort=-1) + + Profile execution of a command and print or save the results. + + This function executes the *command* string using :func:`exec` in the + ``__main__`` module's namespace:: + + exec(command, __main__.__dict__, __main__.__dict__) + + If *filename* is not provided, the function creates a :class:`pstats.Stats` + instance and prints a summary to standard output. If *filename* is + provided, the raw profile data is saved to that file for later analysis + with :mod:`pstats`. + + The *sort* argument specifies the sort order for printed output, accepting + any value recognized by :meth:`pstats.Stats.sort_stats`. + + +.. function:: runctx(command, globals, locals, filename=None, sort=-1) + + Profile execution of a command with explicit namespaces. + + Like :func:`run`, but executes the command with the specified *globals* + and *locals* mappings instead of using the ``__main__`` module's namespace:: + + exec(command, globals, locals) + + +.. class:: Profile(timer=None, timeunit=0.0, subcalls=True, builtins=True) + + A profiler object that collects execution statistics. + + The optional *timer* argument specifies a custom timing function. If not + provided, the profiler uses a platform-appropriate default timer. When + supplying a custom timer, it must return a single number representing the + current time. If the timer returns integers, use *timeunit* to specify the + duration of one time unit (for example, ``0.001`` for milliseconds). + + The *subcalls* argument controls whether the profiler tracks call + relationships between functions. The *builtins* argument controls whether + built-in functions are profiled. + + .. versionchanged:: 3.8 + Added context manager support. + + .. method:: enable() + + Start collecting profiling data. + + .. method:: disable() + + Stop collecting profiling data. + + .. method:: create_stats() + + Stop collecting data and record the results internally as the current + profile. + + .. method:: print_stats(sort=-1) + + Create a :class:`pstats.Stats` object from the current profile and print + the results to standard output. + + The *sort* argument specifies the sorting order. It accepts a single + key or a tuple of keys for multi-level sorting, using the same values + as :meth:`pstats.Stats.sort_stats`. + + .. versionadded:: 3.13 + Support for a tuple of sort keys. + + .. method:: dump_stats(filename) + + Write the current profile data to *filename*. The file can be read by + :class:`pstats.Stats` for later analysis. + + .. method:: run(cmd) + + Profile the command string via :func:`exec`. + + .. method:: runctx(cmd, globals, locals) + + Profile the command string via :func:`exec` with the specified + namespaces. + + .. method:: runcall(func, /, *args, **kwargs) + + Profile a function call. Returns whatever *func* returns:: + + result = pr.runcall(my_function, arg1, arg2, keyword=value) + +.. note:: + + Profiling requires that the profiled code returns normally. If the + interpreter terminates (for example, via :func:`sys.exit`) during + profiling, no results will be available. + + +Using a custom timer +==================== + +The :class:`Profile` class accepts a custom timing function, allowing you to +measure different aspects of execution such as wall-clock time or CPU time. +Pass the timing function to the constructor:: + + pr = profiling.tracing.Profile(my_timer_function) + +The timer function must return a single number representing the current time. +If it returns integers, also specify *timeunit* to indicate the duration of +one unit:: + + # Timer returns time in milliseconds + pr = profiling.tracing.Profile(my_ms_timer, 0.001) + +For best performance, the timer function should be as fast as possible. The +profiler calls it frequently, so timer overhead directly affects profiling +overhead. + +The :mod:`time` module provides several functions suitable for use as custom +timers: + +- :func:`time.perf_counter` for high-resolution wall-clock time +- :func:`time.process_time` for CPU time (excluding sleep) +- :func:`time.monotonic` for monotonic clock time + + +Limitations +=========== + +Deterministic profiling has inherent limitations related to timing accuracy. + +The underlying timer typically has a resolution of about one millisecond. +Measurements cannot be more accurate than this resolution. With enough +measurements, timing errors tend to average out, but individual measurements +may be imprecise. + +There is also latency between when an event occurs and when the profiler +captures the timestamp. Similarly, there is latency after reading the +timestamp before user code resumes. Functions called frequently accumulate +this latency, which can make them appear slower than they actually are. This +error is typically less than one clock tick per call but can become +significant for functions called many times. + +The :mod:`!profiling.tracing` module (and its ``cProfile`` alias) is +implemented as a C extension with low overhead, so these timing issues are +less pronounced than with the deprecated pure Python :mod:`profile` module. + + +.. seealso:: + + :mod:`profiling` + Overview of Python profiling tools and guidance on choosing a profiler. + + :mod:`profiling.sampling` + Statistical sampling profiler for production use. + + :mod:`pstats` + Statistics analysis and formatting for profile data. + + :mod:`profile` + Deprecated pure Python profiler (includes calibration documentation). diff --git a/Doc/library/pstats.rst b/Doc/library/pstats.rst new file mode 100644 index 00000000000..585f17bdb99 --- /dev/null +++ b/Doc/library/pstats.rst @@ -0,0 +1,362 @@ +.. _pstats-module: + +******************************************* +:mod:`!pstats` --- Statistics for profilers +******************************************* + +.. module:: pstats + :synopsis: Statistics object for analyzing profiler output. + +**Source code:** :source:`Lib/pstats.py` + +-------------- + +The :mod:`!pstats` module provides tools for reading, manipulating, and +displaying profiling statistics generated by Python's profilers. It reads +output from both :mod:`profiling.tracing` (deterministic profiler) and +:mod:`profiling.sampling` (statistical profiler). + + +Reading and displaying profile data +=================================== + +The :class:`Stats` class is the primary interface for working with profile +data. It can read statistics from files or directly from a +:class:`~profiling.tracing.Profile` object. + +Load statistics from a file and print a basic report:: + + import pstats + + p = pstats.Stats('profile_output.prof') + p.print_stats() + +The :class:`Stats` object provides methods for sorting and filtering the +data before printing. For example, to see the ten functions with the highest +cumulative time:: + + from pstats import SortKey + + p = pstats.Stats('profile_output.prof') + p.sort_stats(SortKey.CUMULATIVE).print_stats(10) + + +Working with statistics +----------------------- + +The :class:`Stats` class supports method chaining, making it convenient to +perform multiple operations:: + + p = pstats.Stats('restats') + p.strip_dirs().sort_stats(-1).print_stats() + +The :meth:`~Stats.strip_dirs` method removes directory paths from filenames, +making the output more compact. The :meth:`~Stats.sort_stats` method accepts +various keys to control the sort order. + +Different sort keys highlight different aspects of performance:: + + from pstats import SortKey + + # Functions that consume the most cumulative time + p.sort_stats(SortKey.CUMULATIVE).print_stats(10) + + # Functions that consume the most time in their own code + p.sort_stats(SortKey.TIME).print_stats(10) + + # Functions sorted by name + p.sort_stats(SortKey.NAME).print_stats() + + +Filtering output +---------------- + +The :meth:`~Stats.print_stats` method accepts restrictions that filter +which functions are displayed. Restrictions can be integers (limiting the +count), floats between 0 and 1 (selecting a percentage), or strings (matching +function names via regular expression). + +Print only the top 10%:: + + p.print_stats(.1) + +Print only functions whose names contain "init":: + + p.print_stats('init') + +Combine restrictions (they apply sequentially):: + + # Top 10%, then only those containing "init" + p.print_stats(.1, 'init') + + # Functions in files matching "foo:", limited to top 50% + p.sort_stats(SortKey.FILENAME).print_stats('foo:', .5) + + +Analyzing call relationships +---------------------------- + +The :meth:`~Stats.print_callers` method shows which functions called each +displayed function:: + + p.print_callers() + +The :meth:`~Stats.print_callees` method shows the opposite relationship, +listing which functions each displayed function called:: + + p.print_callees() + +Both methods accept the same restriction arguments as :meth:`~Stats.print_stats`. + + +Combining multiple profiles +--------------------------- + +Statistics from multiple profiling runs can be combined into a single +:class:`Stats` object:: + + # Load multiple files at once + p = pstats.Stats('run1.prof', 'run2.prof', 'run3.prof') + + # Or add files incrementally + p = pstats.Stats('run1.prof') + p.add('run2.prof') + p.add('run3.prof') + +When files are combined, statistics for identical functions (same file, line, +and name) are accumulated, giving an aggregate view across all profiling runs. + + +The :class:`!Stats` class +========================= + +.. class:: Stats(*filenames_or_profile, stream=sys.stdout) + + Create a statistics object from profile data. + + The arguments can be filenames (strings or path-like objects) or + :class:`~profiling.tracing.Profile` objects. If multiple sources are + provided, their statistics are combined. + + The *stream* argument specifies where output from :meth:`print_stats` and + related methods is written. It defaults to :data:`sys.stdout`. + + The profile data format is specific to the Python version that created it. + There is no compatibility guarantee between Python versions or between + different profilers. + + .. method:: strip_dirs() + + Remove leading path information from all filenames. + + This method modifies the object in place and returns it for method + chaining. After stripping, the statistics are considered to be in + random order. + + If stripping causes two functions to become indistinguishable (same + filename, line number, and function name), their statistics are + combined into a single entry. + + .. method:: add(*filenames) + + Add profiling data from additional files. + + The files must have been created by the same profiler type. Statistics + for identical functions are accumulated. + + .. method:: dump_stats(filename) + + Save the current statistics to a file. + + The file is created if it does not exist and overwritten if it does. + The saved data can be loaded by creating a new :class:`Stats` object. + + .. method:: sort_stats(*keys) + + Sort the statistics according to the specified criteria. + + Each key can be a string or a :class:`SortKey` enum member. When + multiple keys are provided, later keys break ties in earlier keys. + + Using :class:`SortKey` enum members is preferred over strings as it + provides better error checking:: + + from pstats import SortKey + p.sort_stats(SortKey.CUMULATIVE) + + Valid sort keys: + + +------------------+------------------------+----------------------+ + | String | Enum | Meaning | + +==================+========================+======================+ + | ``'calls'`` | ``SortKey.CALLS`` | call count | + +------------------+------------------------+----------------------+ + | ``'cumulative'`` | ``SortKey.CUMULATIVE`` | cumulative time | + +------------------+------------------------+----------------------+ + | ``'cumtime'`` | N/A | cumulative time | + +------------------+------------------------+----------------------+ + | ``'file'`` | N/A | file name | + +------------------+------------------------+----------------------+ + | ``'filename'`` | ``SortKey.FILENAME`` | file name | + +------------------+------------------------+----------------------+ + | ``'module'`` | N/A | file name | + +------------------+------------------------+----------------------+ + | ``'ncalls'`` | N/A | call count | + +------------------+------------------------+----------------------+ + | ``'pcalls'`` | ``SortKey.PCALLS`` | primitive call count | + +------------------+------------------------+----------------------+ + | ``'line'`` | ``SortKey.LINE`` | line number | + +------------------+------------------------+----------------------+ + | ``'name'`` | ``SortKey.NAME`` | function name | + +------------------+------------------------+----------------------+ + | ``'nfl'`` | ``SortKey.NFL`` | name/file/line | + +------------------+------------------------+----------------------+ + | ``'stdname'`` | ``SortKey.STDNAME`` | standard name | + +------------------+------------------------+----------------------+ + | ``'time'`` | ``SortKey.TIME`` | internal time | + +------------------+------------------------+----------------------+ + | ``'tottime'`` | N/A | internal time | + +------------------+------------------------+----------------------+ + + All sorts on statistics are in descending order (most time consuming + first), while name, file, and line number sorts are ascending + (alphabetical). + + The difference between ``SortKey.NFL`` and ``SortKey.STDNAME`` is that + NFL sorts line numbers numerically while STDNAME sorts them as strings. + ``sort_stats(SortKey.NFL)`` is equivalent to + ``sort_stats(SortKey.NAME, SortKey.FILENAME, SortKey.LINE)``. + + For backward compatibility, the numeric arguments ``-1``, ``0``, ``1``, + and ``2`` are also accepted, meaning ``'stdname'``, ``'calls'``, + ``'time'``, and ``'cumulative'`` respectively. + + .. versionadded:: 3.7 + The :class:`SortKey` enum. + + .. method:: reverse_order() + + Reverse the current sort order. + + By default, the sort direction is chosen appropriately for the sort key + (descending for time-based keys, ascending for name-based keys). This + method inverts that choice. + + .. method:: print_stats(*restrictions) + + Print a report of the profiling statistics. + + The output includes a header line summarizing the data, followed by a + table of function statistics sorted according to the last + :meth:`sort_stats` call. + + Restrictions filter the output. Each restriction is either: + + - An integer: limits output to that many entries + - A float between 0.0 and 1.0: selects that fraction of entries + - A string: matches function names via regular expression + + Restrictions are applied sequentially. For example:: + + print_stats(.1, 'foo:') + + First limits to the top 10%, then filters to functions matching 'foo:'. + + .. method:: print_callers(*restrictions) + + Print the callers of each function in the statistics. + + For each function in the filtered results, shows which functions called + it and how often. + + With :mod:`profiling.tracing` (or ``cProfile``), each caller line + shows three numbers: the number of calls from that caller, and the + total and cumulative times for those specific calls. + + Accepts the same restriction arguments as :meth:`print_stats`. + + .. method:: print_callees(*restrictions) + + Print the functions called by each function in the statistics. + + This is the inverse of :meth:`print_callers`, showing which functions + each listed function called. + + Accepts the same restriction arguments as :meth:`print_stats`. + + .. method:: get_stats_profile() + + Return a ``StatsProfile`` object containing the statistics. + + The returned object provides programmatic access to the profile data, + with function names mapped to ``FunctionProfile`` objects + containing timing and call count information. + + .. versionadded:: 3.9 + + +.. class:: SortKey + + An enumeration of valid sort keys for :meth:`Stats.sort_stats`. + + .. attribute:: CALLS + + Sort by call count. + + .. attribute:: CUMULATIVE + + Sort by cumulative time. + + .. attribute:: FILENAME + + Sort by file name. + + .. attribute:: LINE + + Sort by line number. + + .. attribute:: NAME + + Sort by function name. + + .. attribute:: NFL + + Sort by name, then file, then line number (numeric line sort). + + .. attribute:: PCALLS + + Sort by primitive (non-recursive) call count. + + .. attribute:: STDNAME + + Sort by standard name (string-based line sort). + + .. attribute:: TIME + + Sort by internal time (time in function excluding subcalls). + + +.. _pstats-cli: + +Command-line interface +====================== + +The :mod:`!pstats` module can be invoked as a script to interactively browse +profile data:: + + python -m pstats profile_output.prof + +This opens a line-oriented interface (built on :mod:`cmd`) for examining the +statistics. Type ``help`` at the prompt for available commands. + + +.. seealso:: + + :mod:`profiling` + Overview of Python profiling tools. + + :mod:`profiling.tracing` + Deterministic tracing profiler. + + :mod:`profiling.sampling` + Statistical sampling profiler. diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst index 1a44bb13a84..a7be5779fb2 100644 --- a/Doc/library/pty.rst +++ b/Doc/library/pty.rst @@ -2,17 +2,13 @@ ========================================= .. module:: pty - :platform: Unix :synopsis: Pseudo-Terminal Handling for Unix. -.. moduleauthor:: Steen Lumholt -.. sectionauthor:: Moshe Zadka - **Source code:** :source:`Lib/pty.py` -------------- -The :mod:`pty` module defines operations for handling the pseudo-terminal +The :mod:`!pty` module defines operations for handling the pseudo-terminal concept: starting another process and being able to write to and read from its controlling terminal programmatically. @@ -22,7 +18,7 @@ Pseudo-terminal handling is highly platform dependent. This code is mainly tested on Linux, FreeBSD, and macOS (it is supposed to work on other POSIX platforms but it's not been thoroughly tested). -The :mod:`pty` module defines the following functions: +The :mod:`!pty` module defines the following functions: .. function:: fork() @@ -33,9 +29,14 @@ The :mod:`pty` module defines the following functions: file descriptor connected to the child's controlling terminal (and also to the child's standard input and output). + The returned file descriptor *fd* is :ref:`non-inheritable `. + .. warning:: On macOS the use of this function is unsafe when mixed with using higher-level system APIs, and that includes using :mod:`urllib.request`. + .. versionchanged:: 3.15 + The returned file descriptor is now made non-inheritable. + .. function:: openpty() @@ -88,8 +89,6 @@ The :mod:`pty` module defines the following functions: Example ------- -.. sectionauthor:: Steen Lumholt - The following program acts like the Unix command :manpage:`script(1)`, using a pseudo-terminal to record all input and output of a terminal session in a "typescript". :: diff --git a/Doc/library/pwd.rst b/Doc/library/pwd.rst index e1ff3291213..7691fed2c7c 100644 --- a/Doc/library/pwd.rst +++ b/Doc/library/pwd.rst @@ -2,7 +2,6 @@ ===================================== .. module:: pwd - :platform: Unix :synopsis: The password database (getpwnam() and friends). -------------- diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index 75aa739d100..7aa960de3f2 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -4,16 +4,13 @@ .. module:: py_compile :synopsis: Generate byte-code files from Python source files. -.. sectionauthor:: Fred L. Drake, Jr. -.. documentation based on module docstrings - **Source code:** :source:`Lib/py_compile.py` .. index:: pair: file; byte-code -------------- -The :mod:`py_compile` module provides a function to generate a byte-code file +The :mod:`!py_compile` module provides a function to generate a byte-code file from a source file, and another function used when the module source file is invoked as a script. diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index 5efb11d89dd..ed9fc6d0b5c 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -4,13 +4,11 @@ .. module:: pyclbr :synopsis: Supports information extraction for a Python module browser. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/pyclbr.py` -------------- -The :mod:`pyclbr` module provides limited information about the +The :mod:`!pyclbr` module provides limited information about the functions, classes, and methods defined in a Python-coded module. The information is sufficient to implement a module browser. The information is extracted from the Python source code rather than by diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index e8f153ee1b3..f236eba8457 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -4,9 +4,6 @@ .. module:: pydoc :synopsis: Documentation generator and online help system. -.. moduleauthor:: Ka-Ping Yee -.. sectionauthor:: Ka-Ping Yee - **Source code:** :source:`Lib/pydoc.py` .. index:: diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 5506ac828e5..2e6938b5cf6 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -4,8 +4,6 @@ .. module:: xml.parsers.expat :synopsis: An interface to the Expat non-validating XML parser. -.. moduleauthor:: Paul Prescod - -------------- .. Markup notes: @@ -24,7 +22,7 @@ .. index:: single: Expat -The :mod:`xml.parsers.expat` module is a Python interface to the Expat +The :mod:`!xml.parsers.expat` module is a Python interface to the Expat non-validating XML parser. The module provides a single extension type, :class:`xmlparser`, that represents the current state of an XML parser. After an :class:`xmlparser` object has been created, various attributes of the object @@ -55,7 +53,7 @@ This module provides one exception and one type object: The type of the return values from the :func:`ParserCreate` function. -The :mod:`xml.parsers.expat` module contains two functions: +The :mod:`!xml.parsers.expat` module contains two functions: .. function:: ErrorString(errno) @@ -72,6 +70,13 @@ The :mod:`xml.parsers.expat` module contains two functions: *encoding* [1]_ is given it will override the implicit or explicit encoding of the document. + .. _xmlparser-non-root: + + Parsers created through :func:`!ParserCreate` are called "root" parsers, + in the sense that they do not have any parent parser attached. Non-root + parsers are created by :meth:`parser.ExternalEntityParserCreate + `. + Expat can optionally do XML namespace processing for you, enabled by providing a value for *namespace_separator*. The value must be a one-character string; a :exc:`ValueError` will be raised if the string has an illegal length (``None`` @@ -216,10 +221,10 @@ XMLParser Objects Calling ``SetReparseDeferralEnabled(True)`` allows re-enabling reparse deferral. - Note that :meth:`SetReparseDeferralEnabled` has been backported to some - prior releases of CPython as a security fix. Check for availability of - :meth:`SetReparseDeferralEnabled` using :func:`hasattr` if used in code - running across a variety of Python versions. + :meth:`!SetReparseDeferralEnabled` + has been backported to some prior releases of CPython as a security fix. + Check for availability using :func:`hasattr` if used in code running + across a variety of Python versions. .. versionadded:: 3.13 @@ -231,6 +236,131 @@ XMLParser Objects .. versionadded:: 3.13 +:class:`!xmlparser` objects have the following methods to tune protections +against some common XML vulnerabilities. + +.. method:: xmlparser.SetBillionLaughsAttackProtectionActivationThreshold(threshold, /) + + Sets the number of output bytes needed to activate protection against + `billion laughs`_ attacks. + + The number of output bytes includes amplification from entity expansion + and reading DTD files. + + Parser objects usually have a protection activation threshold of 8 MiB, + but the actual default value depends on the underlying Expat library. + + An :exc:`ExpatError` is raised if this method is called on a + |xml-non-root-parser| parser. + The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset` + should not be used as they may have no special meaning. + + :meth:`!SetBillionLaughsAttackProtectionActivationThreshold` + has been backported to some prior releases of CPython as a security fix. + Check for availability using :func:`hasattr` if used in code running + across a variety of Python versions. + + .. note:: + + Activation thresholds below 4 MiB are known to break support for DITA 1.3 + payload and are hence not recommended. + + .. versionadded:: 3.15 + +.. method:: xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification(max_factor, /) + + Sets the maximum tolerated amplification factor for protection against + `billion laughs`_ attacks. + + The amplification factor is calculated as ``(direct + indirect) / direct`` + while parsing, where ``direct`` is the number of bytes read from + the primary document in parsing and ``indirect`` is the number of + bytes added by expanding entities and reading of external DTD files. + + The *max_factor* value must be a non-NaN :class:`float` value greater than + or equal to 1.0. Peak amplifications of factor 15,000 for the entire payload + and of factor 30,000 in the middle of parsing have been observed with small + benign files in practice. In particular, the activation threshold should be + carefully chosen to avoid false positives. + + Parser objects usually have a maximum amplification factor of 100, + but the actual default value depends on the underlying Expat library. + + An :exc:`ExpatError` is raised if this method is called on a + |xml-non-root-parser| parser or if *max_factor* is outside the valid range. + The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset` + should not be used as they may have no special meaning. + + :meth:`!SetBillionLaughsAttackProtectionMaximumAmplification` + has been backported to some prior releases of CPython as a security fix. + Check for availability using :func:`hasattr` if used in code running + across a variety of Python versions. + + .. note:: + + The maximum amplification factor is only considered if the threshold + that can be adjusted by :meth:`.SetBillionLaughsAttackProtectionActivationThreshold` + is exceeded. + + .. versionadded:: 3.15 + +.. method:: xmlparser.SetAllocTrackerActivationThreshold(threshold, /) + + Sets the number of allocated bytes of dynamic memory needed to activate + protection against disproportionate use of RAM. + + Parser objects usually have an allocation activation threshold of 64 MiB, + but the actual default value depends on the underlying Expat library. + + An :exc:`ExpatError` is raised if this method is called on a + |xml-non-root-parser| parser. + The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset` + should not be used as they may have no special meaning. + + :meth:`!SetAllocTrackerActivationThreshold` + has been backported to some prior releases of CPython as a security fix. + Check for availability using :func:`hasattr` if used in code running + across a variety of Python versions. + + .. versionadded:: 3.15 + +.. method:: xmlparser.SetAllocTrackerMaximumAmplification(max_factor, /) + + Sets the maximum amplification factor between direct input and bytes + of dynamic memory allocated. + + The amplification factor is calculated as ``allocated / direct`` + while parsing, where ``direct`` is the number of bytes read from + the primary document in parsing and ``allocated`` is the number + of bytes of dynamic memory allocated in the parser hierarchy. + + The *max_factor* value must be a non-NaN :class:`float` value greater than + or equal to 1.0. Amplification factors greater than 100.0 can be observed + near the start of parsing even with benign files in practice. In particular, + the activation threshold should be carefully chosen to avoid false positives. + + Parser objects usually have a maximum amplification factor of 100, + but the actual default value depends on the underlying Expat library. + + An :exc:`ExpatError` is raised if this method is called on a + |xml-non-root-parser| parser or if *max_factor* is outside the valid range. + The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset` + should not be used as they may have no special meaning. + + :meth:`!SetAllocTrackerMaximumAmplification` + has been backported to some prior releases of CPython as a security fix. + Check for availability using :func:`hasattr` if used in code running + across a variety of Python versions. + + .. note:: + + The maximum amplification factor is only considered if the threshold + that can be adjusted by :meth:`.SetAllocTrackerActivationThreshold` + is exceeded. + + .. versionadded:: 3.15 + + :class:`xmlparser` objects have the following attributes: @@ -353,7 +483,7 @@ otherwise stated. ...``). The *doctypeName* is provided exactly as presented. The *systemId* and *publicId* parameters give the system and public identifiers if specified, or ``None`` if omitted. *has_internal_subset* will be true if the document - contains and internal document declaration subset. This requires Expat version + contains an internal document declaration subset. This requires Expat version 1.2 or newer. @@ -502,6 +632,15 @@ otherwise stated. .. method:: xmlparser.ExternalEntityRefHandler(context, base, systemId, publicId) + .. warning:: + + Implementing a handler that accesses local files and/or the network + may create a vulnerability to + `external entity attacks `_ + if :class:`xmlparser` is used with user-provided XML content. + Please reflect on your `threat model `_ + before implementing this handler. + Called for references to external entities. *base* is the current base, as set by a previous call to :meth:`SetBase`. The public and system identifiers, *systemId* and *publicId*, are strings if given; if the public identifier is not @@ -525,9 +664,6 @@ otherwise stated. ExpatError Exceptions --------------------- -.. sectionauthor:: Fred L. Drake, Jr. - - :exc:`ExpatError` exceptions have a number of interesting attributes: @@ -611,14 +747,12 @@ Content Model Descriptions .. module:: xml.parsers.expat.model -.. sectionauthor:: Fred L. Drake, Jr. - Content models are described using nested tuples. Each tuple contains four values: the type, the quantifier, the name, and a tuple of children. Children are simply additional content model descriptions. The values of the first two fields are constants defined in the -:mod:`xml.parsers.expat.model` module. These constants can be collected in two +:mod:`!xml.parsers.expat.model` module. These constants can be collected in two groups: the model type group and the quantifier group. The constants in the model type group are: @@ -692,7 +826,7 @@ Expat error constants .. module:: xml.parsers.expat.errors -The following constants are provided in the :mod:`xml.parsers.expat.errors` +The following constants are provided in the :mod:`!xml.parsers.expat.errors` module. These constants are useful in interpreting some of the attributes of the :exc:`ExpatError` exception objects raised when an error has occurred. Since for backwards compatibility reasons, the constants' value is the error @@ -839,7 +973,7 @@ The ``errors`` module has the following attributes: An operation was requested that requires DTD support to be compiled in, but Expat was configured without DTD support. This should never be reported by a - standard build of the :mod:`xml.parsers.expat` module. + standard build of the :mod:`!xml.parsers.expat` module. .. data:: XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING @@ -954,3 +1088,6 @@ The ``errors`` module has the following attributes: not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl and https://www.iana.org/assignments/character-sets/character-sets.xhtml. + +.. _billion laughs: https://en.wikipedia.org/wiki/Billion_laughs_attack +.. |xml-non-root-parser| replace:: :ref:`non-root ` diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index 1b75582f0cf..5ac72ef7604 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -8,7 +8,7 @@ -------------- -The :mod:`queue` module implements multi-producer, multi-consumer queues. +The :mod:`!queue` module implements multi-producer, multi-consumer queues. It is especially useful in threaded programming when information must be exchanged safely between multiple threads. The :class:`Queue` class in this module implements all the required locking semantics. @@ -30,7 +30,7 @@ In addition, the module implements a "simple" specific implementation provides additional guarantees in exchange for the smaller functionality. -The :mod:`queue` module defines the following classes and exceptions: +The :mod:`!queue` module defines the following classes and exceptions: .. class:: Queue(maxsize=0) diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 4e55e301b89..73a37e189dd 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -37,7 +37,7 @@ Class :class:`Random` can also be subclassed if you want to use a different basic generator of your own devising: see the documentation on that class for more details. -The :mod:`random` module also provides the :class:`SystemRandom` class which +The :mod:`!random` module also provides the :class:`SystemRandom` class which uses the system function :func:`os.urandom` to generate random numbers from sources provided by the operating system. @@ -78,7 +78,7 @@ Bookkeeping functions instead of the system time (see the :func:`os.urandom` function for details on availability). - If *a* is an int, it is used directly. + If *a* is an int, its absolute value is used directly. With version 2 (the default), a :class:`str`, :class:`bytes`, or :class:`bytearray` object gets converted to an :class:`int` and all of its bits are used. @@ -410,7 +410,7 @@ Alternative Generator .. class:: Random([seed]) Class that implements the default pseudo-random number generator used by the - :mod:`random` module. + :mod:`!random` module. .. versionchanged:: 3.11 Formerly the *seed* could be any hashable object. Now it is limited to: @@ -634,11 +634,12 @@ from the combinatoric iterators in the :mod:`itertools` module or the :pypi:`more-itertools` project: .. testcode:: + import random - def random_product(*args, repeat=1): - "Random selection from itertools.product(*args, **kwds)" - pools = [tuple(pool) for pool in args] * repeat + def random_product(*iterables, repeat=1): + "Random selection from itertools.product(*iterables, repeat=repeat)" + pools = tuple(map(tuple, iterables)) * repeat return tuple(map(random.choice, pools)) def random_permutation(iterable, r=None): @@ -663,15 +664,89 @@ or the :pypi:`more-itertools` project: return tuple(pool[i] for i in indices) def random_derangement(iterable): - "Choose a permutation where no element is in its original position." + "Choose a permutation where no element stays in its original position." seq = tuple(iterable) if len(seq) < 2: - raise ValueError('derangements require at least two values') - perm = list(seq) + if not seq: + return () + raise IndexError('No derangments to choose from') + perm = list(range(len(seq))) + start = tuple(perm) while True: random.shuffle(perm) - if all(p != q for p, q in zip(seq, perm)): - return tuple(perm) + if all(p != q for p, q in zip(start, perm)): + return tuple([seq[i] for i in perm]) + +.. doctest:: + :hide: + + >>> import random + + + >>> random.seed(8675309) + >>> random_product('ABCDEFG', repeat=5) + ('D', 'B', 'E', 'F', 'E') + + + >>> random.seed(8675309) + >>> random_permutation('ABCDEFG') + ('D', 'B', 'E', 'C', 'G', 'A', 'F') + >>> random_permutation('ABCDEFG', 5) + ('A', 'G', 'D', 'C', 'B') + + + >>> random.seed(8675309) + >>> random_combination('ABCDEFG', 7) + ('A', 'B', 'C', 'D', 'E', 'F', 'G') + >>> random_combination('ABCDEFG', 6) + ('A', 'B', 'C', 'D', 'F', 'G') + >>> random_combination('ABCDEFG', 5) + ('A', 'B', 'C', 'E', 'F') + >>> random_combination('ABCDEFG', 4) + ('B', 'C', 'D', 'G') + >>> random_combination('ABCDEFG', 3) + ('B', 'E', 'G') + >>> random_combination('ABCDEFG', 2) + ('E', 'G') + >>> random_combination('ABCDEFG', 1) + ('C',) + >>> random_combination('ABCDEFG', 0) + () + + + >>> random.seed(8675309) + >>> random_combination_with_replacement('ABCDEFG', 7) + ('B', 'C', 'D', 'E', 'E', 'E', 'G') + >>> random_combination_with_replacement('ABCDEFG', 3) + ('A', 'B', 'E') + >>> random_combination_with_replacement('ABCDEFG', 2) + ('A', 'G') + >>> random_combination_with_replacement('ABCDEFG', 1) + ('E',) + >>> random_combination_with_replacement('ABCDEFG', 0) + () + + + >>> random.seed(8675309) + >>> random_derangement('') + () + >>> random_derangement('A') + Traceback (most recent call last): + ... + IndexError: No derangments to choose from + >>> random_derangement('AB') + ('B', 'A') + >>> random_derangement('ABC') + ('C', 'A', 'B') + >>> random_derangement('ABCD') + ('B', 'A', 'D', 'C') + >>> random_derangement('ABCDE') + ('B', 'C', 'A', 'E', 'D') + >>> # Identical inputs treated as distinct + >>> identical = 20 + >>> random_derangement((10, identical, 30, identical)) + (20, 30, 10, 20) + The default :func:`.random` returns multiples of 2⁻⁵³ in the range *0.0 ≤ x < 1.0*. All such numbers are evenly spaced and are exactly diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 75ebbf11c8e..a46fd424581 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -4,9 +4,6 @@ .. module:: re :synopsis: Regular expression operations. -.. moduleauthor:: Fredrik Lundh -.. sectionauthor:: Andrew M. Kuchling - **Source code:** :source:`Lib/re/` -------------- @@ -49,13 +46,13 @@ fine-tuning parameters. .. seealso:: The third-party :pypi:`regex` module, - which has an API compatible with the standard library :mod:`re` module, + which has an API compatible with the standard library :mod:`!re` module, but offers additional functionality and a more thorough Unicode support. .. _re-syntax: -Regular Expression Syntax +Regular expression syntax ------------------------- A regular expression (or RE) specifies a set of strings that matches it; the @@ -208,7 +205,7 @@ The special characters are: *without* establishing any backtracking points. This is the possessive version of the quantifier above. For example, on the 6-character string ``'aaaaaa'``, ``a{3,5}+aa`` - attempt to match 5 ``'a'`` characters, then, requiring 2 more ``'a'``\ s, + attempts to match 5 ``'a'`` characters, then, requiring 2 more ``'a'``\ s, will need more characters than available and thus fail, while ``a{3,5}aa`` will match with ``a{3,5}`` capturing 5, then 4 ``'a'``\ s by backtracking and then the final 2 ``'a'``\ s are matched by the final @@ -720,7 +717,7 @@ three digits in length. .. _contents-of-module-re: -Module Contents +Module contents --------------- The module defines several functions, constants, and an exception. Some of the @@ -836,8 +833,8 @@ Flags will be conditionally ORed with other flags. Example of use as a default value:: - def myfunc(text, flag=re.NOFLAG): - return re.match(text, flag) + def myfunc(pattern, text, flag=re.NOFLAG): + return re.search(pattern, text, flag) .. versionadded:: 3.11 @@ -893,8 +890,8 @@ Functions Compile a regular expression pattern into a :ref:`regular expression object `, which can be used for matching using its - :func:`~Pattern.match`, :func:`~Pattern.search` and other methods, described - below. + :func:`~Pattern.prefixmatch`, + :func:`~Pattern.search`, and other methods, described below. The expression's behaviour can be modified by specifying a *flags* value. Values can be any of the `flags`_ variables, combined using bitwise OR @@ -903,11 +900,11 @@ Functions The sequence :: prog = re.compile(pattern) - result = prog.match(string) + result = prog.search(string) is equivalent to :: - result = re.match(pattern, string) + result = re.search(pattern, string) but using :func:`re.compile` and saving the resulting regular expression object for reuse is more efficient when the expression will be used several @@ -933,15 +930,17 @@ Functions (the ``|`` operator). -.. function:: match(pattern, string, flags=0) +.. function:: prefixmatch(pattern, string, flags=0) If zero or more characters at the beginning of *string* match the regular expression *pattern*, return a corresponding :class:`~re.Match`. Return ``None`` if the string does not match the pattern; note that this is different from a zero-length match. - Note that even in :const:`MULTILINE` mode, :func:`re.match` will only match - at the beginning of the string and not at the beginning of each line. + .. note:: + + Even in :const:`MULTILINE` mode, this will only match at the + beginning of the string and not at the beginning of each line. If you want to locate a match anywhere in *string*, use :func:`search` instead (see also :ref:`search-vs-match`). @@ -950,6 +949,23 @@ Functions Values can be any of the `flags`_ variables, combined using bitwise OR (the ``|`` operator). + This function now has two names and has long been known as + :func:`~re.match`. Use that name when you need to retain compatibility with + older Python versions. + + .. versionadded:: 3.15 + +.. function:: match(pattern, string, flags=0) + + .. soft-deprecated:: 3.15 + :func:`~re.match` has been :term:`soft deprecated` in favor of + the alternate :func:`~re.prefixmatch` name of this API which is + more explicitly descriptive. Use it to better + express intent. The norm in other languages and regular expression + implementations is to use the term *match* to refer to the behavior of + what Python has always called :func:`~re.search`. + See :ref:`prefixmatch-vs-match`. + .. function:: fullmatch(pattern, string, flags=0) @@ -1234,7 +1250,7 @@ Exceptions .. _re-objects: -Regular Expression Objects +Regular expression objects -------------------------- .. class:: Pattern @@ -1271,24 +1287,44 @@ Regular Expression Objects >>> pattern.search("dog", 1) # No match; search doesn't include the "d" -.. method:: Pattern.match(string[, pos[, endpos]]) +.. method:: Pattern.prefixmatch(string[, pos[, endpos]]) If zero or more characters at the *beginning* of *string* match this regular expression, return a corresponding :class:`~re.Match`. Return ``None`` if the string does not match the pattern; note that this is different from a zero-length match. + Note that even in :const:`MULTILINE` mode, this will only match at the + beginning of the string and not at the beginning of each line. + The optional *pos* and *endpos* parameters have the same meaning as for the :meth:`~Pattern.search` method. :: >>> pattern = re.compile("o") - >>> pattern.match("dog") # No match as "o" is not at the start of "dog". - >>> pattern.match("dog", 1) # Match as "o" is the 2nd character of "dog". + >>> pattern.prefixmatch("dog") # No match as "o" is not at the start of "dog". + >>> pattern.prefixmatch("dog", 1) # Match as "o" is the 2nd character of "dog". If you want to locate a match anywhere in *string*, use :meth:`~Pattern.search` instead (see also :ref:`search-vs-match`). + This method now has two names and has long been known as + :meth:`~Pattern.match`. Use that name when you need to retain compatibility + with older Python versions. + + .. versionadded:: 3.15 + +.. method:: Pattern.match(string[, pos[, endpos]]) + + .. soft-deprecated:: 3.15 + :meth:`~Pattern.match` has been :term:`soft deprecated` in favor of + the alternate :meth:`~Pattern.prefixmatch` name of this API which is + more explicitly descriptive. Use it to + better express intent. The norm in other languages and regular expression + implementations is to use the term *match* to refer to the behavior of + what Python has always called :meth:`~Pattern.search`. + See :ref:`prefixmatch-vs-match`. + .. method:: Pattern.fullmatch(string[, pos[, endpos]]) @@ -1368,7 +1404,7 @@ Regular Expression Objects .. _match-objects: -Match Objects +Match objects ------------- Match objects always have a boolean value of ``True``. @@ -1376,8 +1412,7 @@ Since :meth:`~Pattern.match` and :meth:`~Pattern.search` return ``None`` when there is no match, you can test whether there was a match with a simple ``if`` statement:: - match = re.search(pattern, string) - if match: + if match := re.search(pattern, string): process(match) .. class:: Match @@ -1407,23 +1442,23 @@ when there is no match, you can test whether there was a match with a simple result is a single string; if there are multiple arguments, the result is a tuple with one item per argument. Without arguments, *group1* defaults to zero (the whole match is returned). If a *groupN* argument is zero, the corresponding - return value is the entire matching string; if it is in the inclusive range - [1..99], it is the string matching the corresponding parenthesized group. If a - group number is negative or larger than the number of groups defined in the - pattern, an :exc:`IndexError` exception is raised. If a group is contained in a + return value is the entire matching string; if it is a positive integer, it is + the string matching the corresponding parenthesized group. If a group number is + negative or larger than the number of groups defined in the pattern, an + :exc:`IndexError` exception is raised. If a group is contained in a part of the pattern that did not match, the corresponding result is ``None``. If a group is contained in a part of the pattern that matched multiple times, the last match is returned. :: - >>> m = re.match(r"(\w+) (\w+)", "Isaac Newton, physicist") + >>> m = re.search(r"\A(\w+) (\w+)", "Norwegian Blue, pining for the fjords") >>> m.group(0) # The entire match - 'Isaac Newton' + 'Norwegian Blue' >>> m.group(1) # The first parenthesized subgroup. - 'Isaac' + 'Norwegian' >>> m.group(2) # The second parenthesized subgroup. - 'Newton' + 'Blue' >>> m.group(1, 2) # Multiple arguments give us a tuple. - ('Isaac', 'Newton') + ('Norwegian', 'Blue') If the regular expression uses the ``(?P...)`` syntax, the *groupN* arguments may also be strings identifying groups by their group name. If a @@ -1432,23 +1467,23 @@ when there is no match, you can test whether there was a match with a simple A moderately complicated example:: - >>> m = re.match(r"(?P\w+) (?P\w+)", "Malcolm Reynolds") - >>> m.group('first_name') - 'Malcolm' - >>> m.group('last_name') - 'Reynolds' + >>> m = re.search(r"(?P\w+) (?P\w+)", "killer rabbit") + >>> m.group('adjective') + 'killer' + >>> m.group('animal') + 'rabbit' Named groups can also be referred to by their index:: >>> m.group(1) - 'Malcolm' + 'killer' >>> m.group(2) - 'Reynolds' + 'rabbit' If a group matches multiple times, only the last match is accessible:: - >>> m = re.match(r"(..)+", "a1b2c3") # Matches 3 times. - >>> m.group(1) # Returns only the last match. + >>> m = re.search(r"(..)+", "a1b2c3") # Matches 3 times. + >>> m.group(1) # Returns only the last match. 'c3' @@ -1457,21 +1492,21 @@ when there is no match, you can test whether there was a match with a simple This is identical to ``m.group(g)``. This allows easier access to an individual group from a match:: - >>> m = re.match(r"(\w+) (\w+)", "Isaac Newton, physicist") + >>> m = re.search(r"(\w+) (\w+)", "Norwegian Blue, pining for the fjords") >>> m[0] # The entire match - 'Isaac Newton' + 'Norwegian Blue' >>> m[1] # The first parenthesized subgroup. - 'Isaac' + 'Norwegian' >>> m[2] # The second parenthesized subgroup. - 'Newton' + 'Blue' Named groups are supported as well:: - >>> m = re.match(r"(?P\w+) (?P\w+)", "Isaac Newton") - >>> m['first_name'] - 'Isaac' - >>> m['last_name'] - 'Newton' + >>> m = re.search(r"(?P\w+) (?P\w+)", "killer rabbit") + >>> m['adjective'] + 'killer' + >>> m['animal'] + 'rabbit' .. versionadded:: 3.6 @@ -1484,7 +1519,7 @@ when there is no match, you can test whether there was a match with a simple For example:: - >>> m = re.match(r"(\d+)\.(\d+)", "24.1632") + >>> m = re.search(r"(\d+)\.(\d+)", "24.1632") >>> m.groups() ('24', '1632') @@ -1492,7 +1527,7 @@ when there is no match, you can test whether there was a match with a simple might participate in the match. These groups will default to ``None`` unless the *default* argument is given:: - >>> m = re.match(r"(\d+)\.?(\d+)?", "24") + >>> m = re.search(r"(\d+)\.?(\d+)?", "24") >>> m.groups() # Second group defaults to None. ('24', None) >>> m.groups('0') # Now, the second group defaults to '0'. @@ -1505,9 +1540,9 @@ when there is no match, you can test whether there was a match with a simple the subgroup name. The *default* argument is used for groups that did not participate in the match; it defaults to ``None``. For example:: - >>> m = re.match(r"(?P\w+) (?P\w+)", "Malcolm Reynolds") + >>> m = re.search(r"(?P\w+) (?P\w+)", "killer rabbit") >>> m.groupdict() - {'first_name': 'Malcolm', 'last_name': 'Reynolds'} + {'adjective': 'killer', 'animal': 'rabbit'} .. method:: Match.start([group]) @@ -1588,11 +1623,11 @@ when there is no match, you can test whether there was a match with a simple .. _re-examples: -Regular Expression Examples +Regular expression examples --------------------------- -Checking for a Pair +Checking for a pair ^^^^^^^^^^^^^^^^^^^ In this example, we'll use the following helper function to display match @@ -1610,42 +1645,41 @@ representing the card with that value. To see if a given string is a valid hand, one could do the following:: - >>> valid = re.compile(r"^[a2-9tjqk]{5}$") - >>> displaymatch(valid.match("akt5q")) # Valid. + >>> valid_hand_re = re.compile(r"^[a2-9tjqk]{5}$") + >>> displaymatch(valid_hand_re.search("akt5q")) # Valid. "" - >>> displaymatch(valid.match("akt5e")) # Invalid. - >>> displaymatch(valid.match("akt")) # Invalid. - >>> displaymatch(valid.match("727ak")) # Valid. + >>> displaymatch(valid_hand_re.search("akt5e")) # Invalid. + >>> displaymatch(valid_hand_re.search("akt")) # Invalid. + >>> displaymatch(valid_hand_re.search("727ak")) # Valid. "" That last hand, ``"727ak"``, contained a pair, or two of the same valued cards. To match this with a regular expression, one could use backreferences as such:: - >>> pair = re.compile(r".*(.).*\1") - >>> displaymatch(pair.match("717ak")) # Pair of 7s. + >>> pair_re = re.compile(r".*(.).*\1") + >>> displaymatch(pair_re.prefixmatch("717ak")) # Pair of 7s. "" - >>> displaymatch(pair.match("718ak")) # No pairs. - >>> displaymatch(pair.match("354aa")) # Pair of aces. + >>> displaymatch(pair_re.prefixmatch("718ak")) # No pairs. + >>> displaymatch(pair_re.prefixmatch("354aa")) # Pair of aces. "" To find out what card the pair consists of, one could use the :meth:`~Match.group` method of the match object in the following manner:: - >>> pair = re.compile(r".*(.).*\1") - >>> pair.match("717ak").group(1) + >>> pair_re = re.compile(r".*(.).*\1") + >>> pair_re.prefixmatch("717ak").group(1) '7' - # Error because re.match() returns None, which doesn't have a group() method: - >>> pair.match("718ak").group(1) + # Error because prefixmatch() returns None, which doesn't have a group() method: + >>> pair_re.prefixmatch("718ak").group(1) Traceback (most recent call last): File "", line 1, in - re.match(r".*(.).*\1", "718ak").group(1) + pair_re.prefixmatch("718ak").group(1) AttributeError: 'NoneType' object has no attribute 'group' - >>> pair.match("354aa").group(1) + >>> pair_re.prefixmatch("354aa").group(1) 'a' - Simulating scanf() ^^^^^^^^^^^^^^^^^^ @@ -1679,38 +1713,41 @@ expressions. | ``%x``, ``%X`` | ``[-+]?(0[xX])?[\dA-Fa-f]+`` | +--------------------------------+---------------------------------------------+ -To extract the filename and numbers from a string like :: +To extract the filename and numbers from a string like: + +.. code-block:: text /usr/sbin/sendmail - 0 errors, 4 warnings -you would use a :c:func:`!scanf` format like :: +you would use a :c:func:`!scanf` format like: + +.. code-block:: text %s - %d errors, %d warnings -The equivalent regular expression would be :: +The equivalent regular expression would be: + +.. code-block:: text (\S+) - (\d+) errors, (\d+) warnings .. _search-vs-match: -search() vs. match() -^^^^^^^^^^^^^^^^^^^^ - -.. sectionauthor:: Fred L. Drake, Jr. +search() vs. prefixmatch() +^^^^^^^^^^^^^^^^^^^^^^^^^^ Python offers different primitive operations based on regular expressions: -+ :func:`re.match` checks for a match only at the beginning of the string ++ :func:`re.prefixmatch` checks for a match only at the beginning of the string + :func:`re.search` checks for a match anywhere in the string (this is what Perl does by default) + :func:`re.fullmatch` checks for entire string to be a match - For example:: - >>> re.match("c", "abcdef") # No match - >>> re.search("c", "abcdef") # Match + >>> re.prefixmatch("c", "abcdef") # No match + >>> re.search("c", "abcdef") # Match >>> re.fullmatch("p.*n", "python") # Match @@ -1719,21 +1756,54 @@ For example:: Regular expressions beginning with ``'^'`` can be used with :func:`search` to restrict the match at the beginning of the string:: - >>> re.match("c", "abcdef") # No match - >>> re.search("^c", "abcdef") # No match - >>> re.search("^a", "abcdef") # Match + >>> re.prefixmatch("c", "abcdef") # No match + >>> re.search("^c", "abcdef") # No match + >>> re.search("^a", "abcdef") # Match -Note however that in :const:`MULTILINE` mode :func:`match` only matches at the +Note however that in :const:`MULTILINE` mode :func:`prefixmatch` only matches at the beginning of the string, whereas using :func:`search` with a regular expression beginning with ``'^'`` will match at the beginning of each line. :: - >>> re.match("X", "A\nB\nX", re.MULTILINE) # No match + >>> re.prefixmatch("X", "A\nB\nX", re.MULTILINE) # No match >>> re.search("^X", "A\nB\nX", re.MULTILINE) # Match +.. _prefixmatch-vs-match: -Making a Phonebook +prefixmatch() vs. match() +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Why is the :func:`~re.match` function and method discouraged in +favor of the longer :func:`~re.prefixmatch` spelling? + +Many other languages have gained regex support libraries since regular +expressions were added to Python. However in the most popular of those, they +use the term *match* in their APIs to mean the unanchored behavior provided in +Python by :func:`~re.search`. Thus use of the plain term *match* can be +unclear to those used to other languages when reading or writing code and +not familiar with the Python API's divergence from what otherwise become the +industry norm. + +Quoting from the Zen Of Python (``python3 -m this``): *"Explicit is better than +implicit"*. Anyone reading the name :func:`!prefixmatch` is likely to +understand the intended semantics. When reading :func:`!match` there remains +a seed of doubt about the intended behavior to anyone not already familiar with +this old Python gotcha. + +We **do not** plan to remove the older :func:`!match` name, +as it has been used in code for over 30 years. +It has been :term:`soft deprecated`: +code supporting older versions of Python should continue to use :func:`!match`, +while new code should prefer :func:`!prefixmatch`. + +.. versionadded:: 3.15 + :func:`!prefixmatch` + +.. soft-deprecated:: 3.15 + :func:`!match` + +Making a phonebook ^^^^^^^^^^^^^^^^^^ :func:`split` splits a string into a list delimited by the passed pattern. The @@ -1794,7 +1864,7 @@ house number from the street name: ['Heather', 'Albrecht', '548.326.4584', '919', 'Park Place']] -Text Munging +Text munging ^^^^^^^^^^^^ :func:`sub` replaces every occurrence of a pattern with a string or the @@ -1814,7 +1884,7 @@ in each word of a sentence except for the first and last characters:: 'Pofsroser Aodlambelk, plasee reoprt yuor asnebces potlmrpy.' -Finding all Adverbs +Finding all adverbs ^^^^^^^^^^^^^^^^^^^ :func:`findall` matches *all* occurrences of a pattern, not just the first @@ -1827,7 +1897,7 @@ the following manner:: ['carefully', 'quickly'] -Finding all Adverbs and their Positions +Finding all adverbs and their positions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If one wants more information about all matches of a pattern than the matched @@ -1843,7 +1913,7 @@ to find all of the adverbs *and their positions* in some text, they would use 40-47: quickly -Raw String Notation +Raw string notation ^^^^^^^^^^^^^^^^^^^ Raw string notation (``r"text"``) keeps regular expressions sane. Without it, @@ -1851,9 +1921,9 @@ every backslash (``'\'``) in a regular expression would have to be prefixed with another one to escape it. For example, the two following lines of code are functionally identical:: - >>> re.match(r"\W(.)\1\W", " ff ") + >>> re.search(r"\W(.)\1\W", " ff ") - >>> re.match("\\W(.)\\1\\W", " ff ") + >>> re.search("\\W(.)\\1\\W", " ff ") When one wants to match a literal backslash, it must be escaped in the regular @@ -1861,13 +1931,13 @@ expression. With raw string notation, this means ``r"\\"``. Without raw string notation, one must use ``"\\\\"``, making the following lines of code functionally identical:: - >>> re.match(r"\\", r"\\") + >>> re.search(r"\\", r"\\") - >>> re.match("\\\\", r"\\") + >>> re.search("\\\\", r"\\") -Writing a Tokenizer +Writing a tokenizer ^^^^^^^^^^^^^^^^^^^ A `tokenizer or scanner `_ @@ -1883,7 +1953,7 @@ successive matches:: class Token(NamedTuple): type: str - value: str + value: int | float | str line: int column: int diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index f649fce5efc..234af8d191e 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -2,14 +2,11 @@ =========================================== .. module:: readline - :platform: Unix :synopsis: GNU readline support for Python. -.. sectionauthor:: Skip Montanaro - -------------- -The :mod:`readline` module defines a number of functions to facilitate +The :mod:`!readline` module defines a number of functions to facilitate completion and reading/writing of history files from the Python interpreter. This module can be used directly, or via the :mod:`rlcompleter` module, which supports completion of Python identifiers at the interactive prompt. Settings @@ -26,11 +23,15 @@ Readline library in general. .. include:: ../includes/wasm-mobile-notavail.rst +.. include:: ../includes/optional-module.rst + +.. availability:: Unix. + .. note:: The underlying Readline library API may be implemented by the ``editline`` (``libedit``) library instead of GNU readline. - On macOS the :mod:`readline` module detects which library is being used + On macOS the :mod:`!readline` module detects which library is being used at run time. The configuration file for ``editline`` is different from that @@ -244,6 +245,15 @@ Startup hooks if Python was compiled for a version of the library that supports it. +.. function:: get_pre_input_hook() + + Get the current pre-input hook function, or ``None`` if no pre-input hook + function has been set. This function only exists if Python was compiled + for a version of the library that supports it. + + .. versionadded:: 3.15 + + .. _readline-completion: Completion @@ -253,7 +263,7 @@ The following functions relate to implementing a custom word completion function. This is typically operated by the Tab key, and can suggest and automatically complete a word being typed. By default, Readline is set up to be used by :mod:`rlcompleter` to complete Python identifiers for -the interactive interpreter. If the :mod:`readline` module is to be used +the interactive interpreter. If the :mod:`!readline` module is to be used with a custom completer, a different set of word delimiters should be set. @@ -322,7 +332,7 @@ with a custom completer, a different set of word delimiters should be set. Example ------- -The following example demonstrates how to use the :mod:`readline` module's +The following example demonstrates how to use the :mod:`!readline` module's history reading and writing functions to automatically load and save a history file named :file:`.python_history` from the user's home directory. The code below would normally be executed automatically during interactive sessions @@ -392,3 +402,9 @@ support history save/restore. :: def save_history(self, histfile): readline.set_history_length(1000) readline.write_history_file(histfile) + +.. note:: + + The new :term:`REPL` introduced in version 3.13 doesn't support readline. + However, readline can still be used by setting the :envvar:`PYTHON_BASIC_REPL` + environment variable. diff --git a/Doc/library/reprlib.rst b/Doc/library/reprlib.rst index 28c7855dfee..d269d8bbaa5 100644 --- a/Doc/library/reprlib.rst +++ b/Doc/library/reprlib.rst @@ -4,8 +4,6 @@ .. module:: reprlib :synopsis: Alternate repr() implementation with size limits. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/reprlib.py` -------------- diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index 5bc68fdeff4..561b2976ece 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -2,12 +2,8 @@ =============================================== .. module:: resource - :platform: Unix :synopsis: An interface to provide resource usage information on the current process. -.. moduleauthor:: Jeremy Hylton -.. sectionauthor:: Jeremy Hylton - -------------- This module provides basic mechanisms for measuring and controlling system @@ -51,7 +47,7 @@ this module for those platforms. Constant used to represent the limit for an unlimited resource. - .. versionchanged:: next + .. versionchanged:: 3.15 It is now always positive. Previously, it could be negative, such as -1 or -3. @@ -63,7 +59,7 @@ this module for those platforms. cannot be represented in the ``rlim_t`` value in C. Can be equal to :data:`RLIM_INFINITY`. - .. versionadded:: next + .. versionadded:: 3.15 .. function:: getrlimit(resource) @@ -141,7 +137,7 @@ platform. .. data:: RLIMIT_CPU The maximum amount of processor time (in seconds) that a process can use. If - this limit is exceeded, a :const:`SIGXCPU` signal is sent to the process. (See + this limit is exceeded, a :const:`~signal.SIGXCPU` signal is sent to the process. (See the :mod:`signal` module documentation for information about how to catch this signal and do something useful, e.g. flush open files to disk.) @@ -296,7 +292,7 @@ platform. .. availability:: NetBSD >= 7.0. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: RLIMIT_PIPEBUF @@ -306,7 +302,7 @@ platform. .. availability:: FreeBSD >= 14.2. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: RLIMIT_THREADS @@ -315,7 +311,7 @@ platform. .. availability:: AIX. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: RLIMIT_UMTXP @@ -325,7 +321,7 @@ platform. .. availability:: FreeBSD >= 11. - .. versionadded:: next + .. versionadded:: 3.15 Resource Usage @@ -356,54 +352,54 @@ These functions are used to retrieve resource usage information: print(getrusage(RUSAGE_SELF)) The fields of the return value each describe how a particular system resource - has been used, e.g. amount of time spent running is user mode or number of times + has been used, e.g. amount of time spent running in user mode or number of times the process was swapped out of main memory. Some values are dependent on the - clock tick internal, e.g. the amount of memory the process is using. + clock tick interval, e.g. the amount of memory the process is using. For backward compatibility, the return value is also accessible as a tuple of 16 elements. - The fields :attr:`ru_utime` and :attr:`ru_stime` of the return value are + The fields :attr:`!ru_utime` and :attr:`!ru_stime` of the return value are floating-point values representing the amount of time spent executing in user mode and the amount of time spent executing in system mode, respectively. The remaining values are integers. Consult the :manpage:`getrusage(2)` man page for detailed information about these values. A brief summary is presented here: - +--------+---------------------+---------------------------------------+ - | Index | Field | Resource | - +========+=====================+=======================================+ - | ``0`` | :attr:`ru_utime` | time in user mode (float seconds) | - +--------+---------------------+---------------------------------------+ - | ``1`` | :attr:`ru_stime` | time in system mode (float seconds) | - +--------+---------------------+---------------------------------------+ - | ``2`` | :attr:`ru_maxrss` | maximum resident set size | - +--------+---------------------+---------------------------------------+ - | ``3`` | :attr:`ru_ixrss` | shared memory size | - +--------+---------------------+---------------------------------------+ - | ``4`` | :attr:`ru_idrss` | unshared memory size | - +--------+---------------------+---------------------------------------+ - | ``5`` | :attr:`ru_isrss` | unshared stack size | - +--------+---------------------+---------------------------------------+ - | ``6`` | :attr:`ru_minflt` | page faults not requiring I/O | - +--------+---------------------+---------------------------------------+ - | ``7`` | :attr:`ru_majflt` | page faults requiring I/O | - +--------+---------------------+---------------------------------------+ - | ``8`` | :attr:`ru_nswap` | number of swap outs | - +--------+---------------------+---------------------------------------+ - | ``9`` | :attr:`ru_inblock` | block input operations | - +--------+---------------------+---------------------------------------+ - | ``10`` | :attr:`ru_oublock` | block output operations | - +--------+---------------------+---------------------------------------+ - | ``11`` | :attr:`ru_msgsnd` | messages sent | - +--------+---------------------+---------------------------------------+ - | ``12`` | :attr:`ru_msgrcv` | messages received | - +--------+---------------------+---------------------------------------+ - | ``13`` | :attr:`ru_nsignals` | signals received | - +--------+---------------------+---------------------------------------+ - | ``14`` | :attr:`ru_nvcsw` | voluntary context switches | - +--------+---------------------+---------------------------------------+ - | ``15`` | :attr:`ru_nivcsw` | involuntary context switches | - +--------+---------------------+---------------------------------------+ + +--------+----------------------+---------------------------------------+ + | Index | Field | Resource | + +========+======================+=======================================+ + | ``0`` | :attr:`!ru_utime` | time in user mode (float seconds) | + +--------+----------------------+---------------------------------------+ + | ``1`` | :attr:`!ru_stime` | time in system mode (float seconds) | + +--------+----------------------+---------------------------------------+ + | ``2`` | :attr:`!ru_maxrss` | maximum resident set size | + +--------+----------------------+---------------------------------------+ + | ``3`` | :attr:`!ru_ixrss` | shared memory size | + +--------+----------------------+---------------------------------------+ + | ``4`` | :attr:`!ru_idrss` | unshared memory size | + +--------+----------------------+---------------------------------------+ + | ``5`` | :attr:`!ru_isrss` | unshared stack size | + +--------+----------------------+---------------------------------------+ + | ``6`` | :attr:`!ru_minflt` | page faults not requiring I/O | + +--------+----------------------+---------------------------------------+ + | ``7`` | :attr:`!ru_majflt` | page faults requiring I/O | + +--------+----------------------+---------------------------------------+ + | ``8`` | :attr:`!ru_nswap` | number of swap outs | + +--------+----------------------+---------------------------------------+ + | ``9`` | :attr:`!ru_inblock` | block input operations | + +--------+----------------------+---------------------------------------+ + | ``10`` | :attr:`!ru_oublock` | block output operations | + +--------+----------------------+---------------------------------------+ + | ``11`` | :attr:`!ru_msgsnd` | messages sent | + +--------+----------------------+---------------------------------------+ + | ``12`` | :attr:`!ru_msgrcv` | messages received | + +--------+----------------------+---------------------------------------+ + | ``13`` | :attr:`!ru_nsignals` | signals received | + +--------+----------------------+---------------------------------------+ + | ``14`` | :attr:`!ru_nvcsw` | voluntary context switches | + +--------+----------------------+---------------------------------------+ + | ``15`` | :attr:`!ru_nivcsw` | involuntary context switches | + +--------+----------------------+---------------------------------------+ This function will raise a :exc:`ValueError` if an invalid *who* parameter is specified. It may also raise :exc:`error` exception in unusual circumstances. diff --git a/Doc/library/rlcompleter.rst b/Doc/library/rlcompleter.rst index 91779feb525..2acd1df3c49 100644 --- a/Doc/library/rlcompleter.rst +++ b/Doc/library/rlcompleter.rst @@ -4,8 +4,6 @@ .. module:: rlcompleter :synopsis: Python identifier completion, suitable for the GNU readline library. -.. sectionauthor:: Moshe Zadka - **Source code:** :source:`Lib/rlcompleter.py` -------------- diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index b07ec6e93f8..536b5980f86 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -4,13 +4,11 @@ .. module:: runpy :synopsis: Locate and run Python modules without importing them first. -.. moduleauthor:: Nick Coghlan - **Source code:** :source:`Lib/runpy.py` -------------- -The :mod:`runpy` module is used to locate and run Python modules without +The :mod:`!runpy` module is used to locate and run Python modules without importing them first. Its main use is to implement the :option:`-m` command line switch that allows scripts to be located using the Python module namespace rather than the filesystem. @@ -20,11 +18,11 @@ current process, and any side effects (such as cached imports of other modules) will remain in place after the functions have returned. Furthermore, any functions and classes defined by the executed code are not -guaranteed to work correctly after a :mod:`runpy` function has returned. +guaranteed to work correctly after a :mod:`!runpy` function has returned. If that limitation is not acceptable for a given use case, :mod:`importlib` is likely to be a more suitable choice than this module. -The :mod:`runpy` module provides two functions: +The :mod:`!runpy` module provides two functions: .. function:: run_module(mod_name, init_globals=None, run_name=None, alter_sys=False) @@ -50,10 +48,10 @@ The :mod:`runpy` module provides two functions: overridden by :func:`run_module`. The special global variables ``__name__``, ``__spec__``, ``__file__``, - ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals - dictionary before the module code is executed. (Note that this is a - minimal set of variables - other variables may be set implicitly as an - interpreter implementation detail.) + ``__loader__`` and ``__package__`` are set in the globals dictionary before + the module code is executed. (Note that this is a minimal set of variables - + other variables may be set implicitly as an interpreter implementation + detail.) ``__name__`` is set to *run_name* if this optional argument is not :const:`None`, to ``mod_name + '.__main__'`` if the named module is a @@ -63,7 +61,7 @@ The :mod:`runpy` module provides two functions: module (that is, ``__spec__.name`` will always be *mod_name* or ``mod_name + '.__main__'``, never *run_name*). - ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` are + ``__file__``, ``__loader__`` and ``__package__`` are :ref:`set as normal ` based on the module spec. If the argument *alter_sys* is supplied and evaluates to :const:`True`, @@ -98,6 +96,9 @@ The :mod:`runpy` module provides two functions: ``__package__`` are deprecated. See :class:`~importlib.machinery.ModuleSpec` for alternatives. + .. versionchanged:: 3.15 + ``__cached__`` is no longer set. + .. function:: run_path(path_name, init_globals=None, run_name=None) .. index:: @@ -125,23 +126,23 @@ The :mod:`runpy` module provides two functions: overridden by :func:`run_path`. The special global variables ``__name__``, ``__spec__``, ``__file__``, - ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals - dictionary before the module code is executed. (Note that this is a - minimal set of variables - other variables may be set implicitly as an - interpreter implementation detail.) + ``__loader__`` and ``__package__`` are set in the globals dictionary before + the module code is executed. (Note that this is a minimal set of variables - + other variables may be set implicitly as an interpreter implementation + detail.) ``__name__`` is set to *run_name* if this optional argument is not :const:`None` and to ``''`` otherwise. If *file_path* directly references a script file (whether as source or as precompiled byte code), then ``__file__`` will be set to - *file_path*, and ``__spec__``, ``__cached__``, ``__loader__`` and + *file_path*, and ``__spec__``, ``__loader__`` and ``__package__`` will all be set to :const:`None`. If *file_path* is a reference to a valid :data:`sys.path` entry, then ``__spec__`` will be set appropriately for the imported :mod:`__main__` module (that is, ``__spec__.name`` will always be ``__main__``). - ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` will be + ``__file__``, ``__loader__`` and ``__package__`` will be :ref:`set as normal ` based on the module spec. A number of alterations are also made to the :mod:`sys` module. Firstly, @@ -173,6 +174,9 @@ The :mod:`runpy` module provides two functions: The setting of ``__cached__``, ``__loader__``, and ``__package__`` are deprecated. + .. versionchanged:: 3.15 + ``__cached__`` is no longer set. + .. seealso:: :pep:`338` -- Executing modules as scripts diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 517dbe8c321..70541c5f3cb 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -4,15 +4,13 @@ .. module:: sched :synopsis: General purpose event scheduler. -.. sectionauthor:: Moshe Zadka - **Source code:** :source:`Lib/sched.py` .. index:: single: event scheduling -------------- -The :mod:`sched` module defines a class which implements a general purpose event +The :mod:`!sched` module defines a class which implements a general purpose event scheduler: .. class:: scheduler(timefunc=time.monotonic, delayfunc=time.sleep) diff --git a/Doc/library/secrets.rst b/Doc/library/secrets.rst index 75dafc54d40..3b5b57fb1c2 100644 --- a/Doc/library/secrets.rst +++ b/Doc/library/secrets.rst @@ -4,8 +4,6 @@ .. module:: secrets :synopsis: Generate secure random numbers for managing secrets. -.. moduleauthor:: Steven D'Aprano -.. sectionauthor:: Steven D'Aprano .. versionadded:: 3.6 .. testsetup:: @@ -17,11 +15,11 @@ ------------- -The :mod:`secrets` module is used for generating cryptographically strong +The :mod:`!secrets` module is used for generating cryptographically strong random numbers suitable for managing data such as passwords, account authentication, security tokens, and related secrets. -In particular, :mod:`secrets` should be used in preference to the +In particular, :mod:`!secrets` should be used in preference to the default pseudo-random number generator in the :mod:`random` module, which is designed for modelling and simulation, not security or cryptography. @@ -33,7 +31,7 @@ is designed for modelling and simulation, not security or cryptography. Random numbers -------------- -The :mod:`secrets` module provides access to the most secure source of +The :mod:`!secrets` module provides access to the most secure source of randomness that your operating system provides. .. class:: SystemRandom @@ -58,43 +56,48 @@ randomness that your operating system provides. Generating tokens ----------------- -The :mod:`secrets` module provides functions for generating secure +The :mod:`!secrets` module provides functions for generating secure tokens, suitable for applications such as password resets, hard-to-guess URLs, and similar. -.. function:: token_bytes([nbytes=None]) +.. function:: token_bytes(nbytes=None) Return a random byte string containing *nbytes* number of bytes. - If *nbytes* is ``None`` or not supplied, a reasonable default is - used. + + If *nbytes* is not specified or ``None``, :const:`DEFAULT_ENTROPY` + is used instead. .. doctest:: - >>> token_bytes(16) #doctest:+SKIP + >>> token_bytes(16) # doctest: +SKIP b'\xebr\x17D*t\xae\xd4\xe3S\xb6\xe2\xebP1\x8b' -.. function:: token_hex([nbytes=None]) +.. function:: token_hex(nbytes=None) Return a random text string, in hexadecimal. The string has *nbytes* - random bytes, each byte converted to two hex digits. If *nbytes* is - ``None`` or not supplied, a reasonable default is used. + random bytes, each byte converted to two hex digits. + + If *nbytes* is not specified or ``None``, :const:`DEFAULT_ENTROPY` + is used instead. .. doctest:: - >>> token_hex(16) #doctest:+SKIP + >>> token_hex(16) # doctest: +SKIP 'f9bf78b9a18ce6d46a0cd2b0b86df9da' -.. function:: token_urlsafe([nbytes=None]) +.. function:: token_urlsafe(nbytes=None) Return a random URL-safe text string, containing *nbytes* random bytes. The text is Base64 encoded, so on average each byte results - in approximately 1.3 characters. If *nbytes* is ``None`` or not - supplied, a reasonable default is used. + in approximately 1.3 characters. + + If *nbytes* is not specified or ``None``, :const:`DEFAULT_ENTROPY` + is used instead. .. doctest:: - >>> token_urlsafe(16) #doctest:+SKIP + >>> token_urlsafe(16) # doctest: +SKIP 'Drmhze6EPcv0fN_81Bj-nA' @@ -107,7 +110,7 @@ tokens need to have sufficient randomness. Unfortunately, what is considered sufficient will necessarily increase as computers get more powerful and able to make more guesses in a shorter period. As of 2015, it is believed that 32 bytes (256 bits) of randomness is sufficient for -the typical use-case expected for the :mod:`secrets` module. +the typical use-case expected for the :mod:`!secrets` module. For those who want to manage their own token length, you can explicitly specify how much randomness is used for tokens by giving an :class:`int` @@ -115,11 +118,13 @@ argument to the various ``token_*`` functions. That argument is taken as the number of bytes of randomness to use. Otherwise, if no argument is provided, or if the argument is ``None``, -the ``token_*`` functions will use a reasonable default instead. +the ``token_*`` functions use :const:`DEFAULT_ENTROPY` instead. -.. note:: +.. data:: DEFAULT_ENTROPY - That default is subject to change at any time, including during + Default number of bytes of randomness used by the ``token_*`` functions. + + The exact value is subject to change at any time, including during maintenance releases. @@ -139,7 +144,7 @@ Other functions Recipes and best practices -------------------------- -This section shows recipes and best practices for using :mod:`secrets` +This section shows recipes and best practices for using :mod:`!secrets` to manage a basic level of security. Generate an eight-character alphanumeric password: diff --git a/Doc/library/select.rst b/Doc/library/select.rst index d2094283d54..09563af14d0 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -18,7 +18,7 @@ it was last read. .. note:: The :mod:`selectors` module allows high-level and efficient I/O - multiplexing, built upon the :mod:`select` module primitives. Users are + multiplexing, built upon the :mod:`!select` module primitives. Users are encouraged to use the :mod:`selectors` module instead, unless they want precise control over the OS-level primitives used. @@ -62,7 +62,7 @@ The module defines the following: *sizehint* informs epoll about the expected number of events to be registered. It must be positive, or ``-1`` to use the default. It is only - used on older systems where :c:func:`!epoll_create1` is not available; + used on older systems where :manpage:`epoll_create1(2)` is not available; otherwise it has no effect (though its value is still checked). *flags* is deprecated and completely ignored. However, when supplied, its @@ -89,6 +89,11 @@ The module defines the following: The *flags* parameter. ``select.EPOLL_CLOEXEC`` is used by default now. Use :func:`os.set_inheritable` to make the file descriptor inheritable. + .. versionchanged:: 3.15 + + When CPython is built, this function may be disabled using + :option:`--disable-epoll`. + .. function:: poll() @@ -115,7 +120,7 @@ The module defines the following: :ref:`kevent-objects` below for the methods supported by kevent objects. -.. function:: select(rlist, wlist, xlist[, timeout]) +.. function:: select(rlist, wlist, xlist, timeout=None) This is a straightforward interface to the Unix :c:func:`!select` system call. The first three arguments are iterables of 'waitable objects': either @@ -129,8 +134,9 @@ The module defines the following: Empty iterables are allowed, but acceptance of three empty iterables is platform-dependent. (It is known to work on Unix but not on Windows.) The - optional *timeout* argument specifies a time-out as a floating-point number - in seconds. When the *timeout* argument is omitted the function blocks until + optional *timeout* argument specifies a time-out in seconds; it may be + a non-integer to specify fractions of seconds. + When the *timeout* argument is omitted or ``None``, the function blocks until at least one file descriptor is ready. A time-out value of zero specifies a poll and never blocks. @@ -164,13 +170,16 @@ The module defines the following: :pep:`475` for the rationale), instead of raising :exc:`InterruptedError`. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. data:: PIPE_BUF The minimum number of bytes which can be written without blocking to a pipe when the pipe has been reported as ready for writing by :func:`~select.select`, :func:`!poll` or another interface in this module. This doesn't apply - to other kind of file-like objects such as sockets. + to other kinds of file-like objects such as sockets. This value is guaranteed by POSIX to be at least 512. @@ -222,7 +231,7 @@ object. implement :meth:`!fileno`, so they can also be used as the argument. *eventmask* is an optional bitmask describing the type of events you want to - check for. The constants are the same that with :c:func:`!poll` + check for. The constants are the same as with :c:func:`!poll` object. The default value is a combination of the constants :const:`POLLIN`, :const:`POLLPRI`, and :const:`POLLOUT`. @@ -237,7 +246,7 @@ object. .. method:: devpoll.modify(fd[, eventmask]) This method does an :meth:`unregister` followed by a - :meth:`register`. It is (a bit) more efficient that doing the same + :meth:`register`. It is (a bit) more efficient than doing the same explicitly. @@ -270,6 +279,9 @@ object. :pep:`475` for the rationale), instead of raising :exc:`InterruptedError`. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. _epoll-objects: @@ -368,7 +380,9 @@ Edge and Level Trigger Polling (epoll) Objects .. method:: epoll.poll(timeout=None, maxevents=-1) - Wait for events. timeout in seconds (float) + Wait for events. + If *timeout* is given, it specifies the length of time in seconds + (may be non-integer) which the system will wait for events before returning. .. versionchanged:: 3.5 The function is now retried with a recomputed timeout when interrupted by @@ -376,6 +390,9 @@ Edge and Level Trigger Polling (epoll) Objects :pep:`475` for the rationale), instead of raising :exc:`InterruptedError`. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. _poll-objects: @@ -464,6 +481,11 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w :pep:`475` for the rationale), instead of raising :exc:`InterruptedError`. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + If ``ppoll()`` function is available, *timeout* has a resolution + of ``1`` ns (``1e-6`` ms) instead of ``1`` ms. + .. _kqueue-objects: @@ -496,7 +518,7 @@ Kqueue Objects - changelist must be an iterable of kevent objects or ``None`` - max_events must be 0 or a positive integer - - timeout in seconds (floats possible); the default is ``None``, + - timeout in seconds (non-integers are possible); the default is ``None``, to wait forever .. versionchanged:: 3.5 @@ -505,6 +527,9 @@ Kqueue Objects :pep:`475` for the rationale), instead of raising :exc:`InterruptedError`. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. _kevent-objects: @@ -560,9 +585,9 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +---------------------------+---------------------------------------------+ | :const:`KQ_EV_DELETE` | Removes an event from the queue | +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ENABLE` | Permitscontrol() to returns the event | + | :const:`KQ_EV_ENABLE` | Permits control() to return the event | +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DISABLE` | Disablesevent | + | :const:`KQ_EV_DISABLE` | Disables event | +---------------------------+---------------------------------------------+ | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence | +---------------------------+---------------------------------------------+ diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst index ee556f1f3ce..2d523a9d2ea 100644 --- a/Doc/library/selectors.rst +++ b/Doc/library/selectors.rst @@ -54,7 +54,7 @@ Classes hierarchy:: In the following, *events* is a bitwise mask indicating which I/O events should -be waited for on a given file object. It can be a combination of the modules +be waited for on a given file object. It can be a combination of the module's constants below: +-----------------------+-----------------------------------------------+ diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index b88fe4157bd..bd3d56f6af5 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -42,7 +42,7 @@ lots of shared sub-objects. The keys are ordinary strings. determine which accessed entries are mutable, nor which ones were actually mutated). - By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` + By default, :mod:`!shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` for serializing and deserializing. This can be changed by supplying *serializer* and *deserializer*, respectively. @@ -51,7 +51,7 @@ lots of shared sub-objects. The keys are ordinary strings. :term:`bytes-like object`; the *protocol* value may be ignored by the serializer. - The *deserializer* argument must be callable which takes a serialized object + The *deserializer* argument must be a callable which takes a serialized object given as a :class:`bytes` object and returns the corresponding object. A :exc:`ShelveError` is raised if *serializer* is given but *deserializer* @@ -64,7 +64,7 @@ lots of shared sub-objects. The keys are ordinary strings. .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. - .. versionchanged:: next + .. versionchanged:: 3.15 Accepts custom *serializer* and *deserializer* functions in place of :func:`pickle.dumps` and :func:`pickle.loads`. @@ -81,11 +81,11 @@ lots of shared sub-objects. The keys are ordinary strings. .. warning:: - Because the :mod:`shelve` module is backed by :mod:`pickle`, it is insecure + Because the :mod:`!shelve` module is backed by :mod:`pickle`, it is insecure to load a shelf from an untrusted source. Like with pickle, loading a shelf can execute arbitrary code. -Shelf objects support most of methods and operations supported by dictionaries +Shelf objects support most of the methods and operations supported by dictionaries (except copying, constructors and operators ``|`` and ``|=``). This eases the transition from dictionary based scripts to those requiring persistent storage. @@ -103,7 +103,7 @@ Two additional methods are supported: Calls :meth:`sync` and attempts to shrink space used on disk by removing empty space resulting from deletions. - .. versionadded:: next + .. versionadded:: 3.15 .. method:: Shelf.close() @@ -133,7 +133,7 @@ Restrictions database should be fairly small, and in rare cases key collisions may cause the database to refuse updates. -* The :mod:`shelve` module does not support *concurrent* read/write access to +* The :mod:`!shelve` module does not support *concurrent* read/write access to shelved objects. (Multiple simultaneous read accesses are safe.) When a program has a shelf open for writing, no other program should have it open for reading or writing. Unix file locking can be used to solve this, but this @@ -185,7 +185,7 @@ Restrictions :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. - .. versionchanged:: next + .. versionchanged:: 3.15 Added the *serializer* and *deserializer* parameters. @@ -204,7 +204,7 @@ Restrictions optional *protocol*, *writeback*, *keyencoding*, *serializer* and *deserializer* parameters have the same interpretation as in :func:`~shelve.open`. - .. versionchanged:: next + .. versionchanged:: 3.15 Added the *serializer* and *deserializer* parameters. @@ -220,7 +220,7 @@ Restrictions and *deserializer* parameters have the same interpretation as in :func:`~shelve.open`. - .. versionchanged:: next + .. versionchanged:: 3.15 Added the *serializer* and *deserializer* parameters. @@ -274,7 +274,7 @@ Exceptions The *deserializer* and *serializer* arguments must be given together. - .. versionadded:: next + .. versionadded:: 3.15 .. seealso:: @@ -283,5 +283,5 @@ Exceptions Generic interface to ``dbm``-style databases. Module :mod:`pickle` - Object serialization used by :mod:`shelve`. + Object serialization used by :mod:`!shelve`. diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index a96f0864dc1..2ab12f2f6f9 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -4,11 +4,6 @@ .. module:: shlex :synopsis: Simple lexical analysis for Unix shell-like languages. -.. moduleauthor:: Eric S. Raymond -.. moduleauthor:: Gustavo Niemeyer -.. sectionauthor:: Eric S. Raymond -.. sectionauthor:: Gustavo Niemeyer - **Source code:** :source:`Lib/shlex.py` -------------- @@ -18,7 +13,7 @@ simple syntaxes resembling that of the Unix shell. This will often be useful for writing minilanguages, (for example, in run control files for Python applications) or for parsing quoted strings. -The :mod:`shlex` module defines the following functions: +The :mod:`!shlex` module defines the following functions: .. function:: split(s, comments=False, posix=True) @@ -98,7 +93,7 @@ The :mod:`shlex` module defines the following functions: .. versionadded:: 3.3 -The :mod:`shlex` module defines the following class: +The :mod:`!shlex` module defines the following class: .. class:: shlex(instream=None, infile=None, posix=False, punctuation_chars=False) @@ -214,7 +209,7 @@ A :class:`~shlex.shlex` instance has the following methods: with the name of the current source file and the ``%d`` with the current input line number (the optional arguments can be used to override these). - This convenience is provided to encourage :mod:`shlex` users to generate error + This convenience is provided to encourage :mod:`!shlex` users to generate error messages in the standard, parseable format understood by Emacs and other Unix tools. @@ -343,7 +338,7 @@ variables which either control lexical analysis or can be used for debugging: Parsing Rules ------------- -When operating in non-POSIX mode, :class:`~shlex.shlex` will try to obey to the +When operating in non-POSIX mode, :class:`~shlex.shlex` will try to obey the following rules. * Quote characters are not recognized within words (``Do"Not"Separate`` is @@ -366,7 +361,7 @@ following rules. * It's not possible to parse empty strings, even if quoted. -When operating in POSIX mode, :class:`~shlex.shlex` will try to obey to the +When operating in POSIX mode, :class:`~shlex.shlex` will try to obey the following parsing rules. * Quotes are stripped out, and do not separate words (``"Do"Not"Separate"`` is @@ -382,7 +377,7 @@ following parsing rules. * Enclosing characters in quotes which are part of :attr:`~shlex.escapedquotes` (e.g. ``'"'``) preserves the literal value of all characters within the quotes, with the exception of the characters - mentioned in :attr:`~shlex.escape`. The escape characters retain its + mentioned in :attr:`~shlex.escape`. The escape characters retain their special meaning only when followed by the quote in use, or the escape character itself. Otherwise the escape character will be considered a normal character. diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 3a4631e7c65..d289ba58c24 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -4,9 +4,6 @@ .. module:: shutil :synopsis: High-level file operations, including copying. -.. sectionauthor:: Fred L. Drake, Jr. -.. partly based on the docstrings - **Source code:** :source:`Lib/shutil.py` .. index:: @@ -15,7 +12,7 @@ -------------- -The :mod:`shutil` module offers a number of high-level operations on files and +The :mod:`!shutil` module offers a number of high-level operations on files and collections of files. In particular, functions are provided which support file copying and removal. For operations on individual files, see also the :mod:`os` module. @@ -515,7 +512,7 @@ Directory and files operations .. exception:: Error - This exception collects exceptions that are raised during a multi-file + Subclass of :exc:`OSError` collecting exceptions raised during a multi-file operation. For :func:`copytree`, the exception argument is a list of 3-tuples (*srcname*, *dstname*, *exception*). @@ -540,10 +537,12 @@ On Solaris :func:`os.sendfile` is used. On Windows :func:`shutil.copyfile` uses a bigger default buffer size (1 MiB instead of 64 KiB) and a :func:`memoryview`-based variant of -:func:`shutil.copyfileobj` is used. +:func:`shutil.copyfileobj` is used, which still reads and writes in a loop. +:func:`shutil.copy2` uses the native ``CopyFile2`` call on Windows, which is the most +efficient method, supports copy-on-write, and preserves metadata. If the fast-copy operation fails and no data was written in the destination -file then shutil will silently fallback on using less efficient +file then shutil will silently fall back to less efficient :func:`copyfileobj` function internally. .. versionchanged:: 3.8 @@ -619,8 +618,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Create an archive file (such as zip or tar) and return its name. - *base_name* is the name of the file to create, including the path, minus - any format-specific extension. + *base_name* is a string or :term:`path-like object` specifying the name of + the file to create, including the path, minus any format-specific extension. *format* is the archive format: one of "zip" (if the :mod:`zlib` module is available), "tar", "gztar" (if the @@ -628,13 +627,14 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. available), "xztar" (if the :mod:`lzma` module is available), or "zstdtar" (if the :mod:`compression.zstd` module is available). - *root_dir* is a directory that will be the root directory of the - archive, all paths in the archive will be relative to it; for example, - we typically chdir into *root_dir* before creating the archive. + *root_dir* is a string or :term:`path-like object` specifying a directory + that will be the root directory of the archive, all paths in the archive + will be relative to it; for example, we typically chdir into *root_dir* + before creating the archive. - *base_dir* is the directory where we start archiving from; - i.e. *base_dir* will be the common prefix of all files and - directories in the archive. *base_dir* must be given relative + *base_dir* is a string or :term:`path-like object` specifying a directory + where we start archiving from; i.e. *base_dir* will be the common prefix of + all files and directories in the archive. *base_dir* must be given relative to *root_dir*. See :ref:`shutil-archiving-example-with-basedir` for how to use *base_dir* and *root_dir* together. @@ -669,12 +669,16 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. This function is now made thread-safe during creation of standard ``.zip`` and tar archives. + .. versionchanged:: 3.15 + Accepts a :term:`path-like object` for *base_name*, *root_dir* and + *base_dir*. + .. function:: get_archive_formats() Return a list of supported formats for archiving. Each element of the returned sequence is a tuple ``(name, description)``. - By default :mod:`shutil` provides these formats: + By default :mod:`!shutil` provides these formats: - *zip*: ZIP file (if the :mod:`zlib` module is available). - *tar*: Uncompressed tar file. Uses POSIX.1-2001 pax format for new archives. @@ -791,7 +795,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Each element of the returned sequence is a tuple ``(name, extensions, description)``. - By default :mod:`shutil` provides these formats: + By default :mod:`!shutil` provides these formats: - *zip*: ZIP file (unpacking compressed files works only if the corresponding module is available). diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index b0307d3dea1..12ad45f557e 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -36,7 +36,7 @@ Execution of Python signal handlers A Python signal handler does not get executed inside the low-level (C) signal handler. Instead, the low-level signal handler sets a flag which tells the :term:`virtual machine` to execute the corresponding Python signal handler -at a later point(for example at the next :term:`bytecode` instruction). +at a later point (for example, at the next :term:`bytecode` instruction). This has consequences: * It makes little sense to catch synchronous errors like :const:`SIGFPE` or @@ -68,6 +68,11 @@ the synchronization primitives from the :mod:`threading` module instead. Besides, only the main thread of the main interpreter is allowed to set a new signal handler. +.. warning:: + + Synchronization primitives such as :class:`threading.Lock` should not be used + within signal handlers. Doing so can lead to unexpected deadlocks. + Module contents --------------- @@ -92,13 +97,13 @@ The signal module defines three enums: .. class:: Handlers - :class:`enum.IntEnum` collection the constants :const:`SIG_DFL` and :const:`SIG_IGN`. + :class:`enum.IntEnum` collection of the constants :const:`SIG_DFL` and :const:`SIG_IGN`. .. versionadded:: 3.5 .. class:: Sigmasks - :class:`enum.IntEnum` collection the constants :const:`SIG_BLOCK`, :const:`SIG_UNBLOCK` and :const:`SIG_SETMASK`. + :class:`enum.IntEnum` collection of the constants :const:`SIG_BLOCK`, :const:`SIG_UNBLOCK` and :const:`SIG_SETMASK`. .. availability:: Unix. @@ -108,7 +113,7 @@ The signal module defines three enums: .. versionadded:: 3.5 -The variables defined in the :mod:`signal` module are: +The variables defined in the :mod:`!signal` module are: .. data:: SIG_DFL @@ -205,10 +210,28 @@ The variables defined in the :mod:`signal` module are: .. availability:: Unix. +.. data:: SIGPROF + + Profiling timer expired. + + .. availability:: Unix. + +.. data:: SIGQUIT + + Terminal quit signal. + + .. availability:: Unix. + .. data:: SIGSEGV Segmentation fault: invalid memory reference. +.. data:: SIGSTOP + + Stop executing (cannot be caught or ignored). + + .. availability:: Unix. + .. data:: SIGSTKFLT Stack fault on coprocessor. The Linux kernel does not raise this signal: it @@ -237,18 +260,30 @@ The variables defined in the :mod:`signal` module are: .. availability:: Unix. +.. data:: SIGVTALRM + + Virtual timer expired. + + .. availability:: Unix. + .. data:: SIGWINCH Window resize signal. .. availability:: Unix. +.. data:: SIGXCPU + + CPU time limit exceeded. + + .. availability:: Unix. + .. data:: SIG* All the signal numbers are defined symbolically. For example, the hangup signal is defined as :const:`signal.SIGHUP`; the variable names are identical to the names used in C programs, as found in ````. The Unix man page for - ':c:func:`signal`' lists the existing signals (on some systems this is + '``signal``' lists the existing signals (on some systems this is :manpage:`signal(2)`, on others the list is in :manpage:`signal(7)`). Note that not all systems define the same set of signal names; only those names defined by the system are defined by this module. @@ -322,7 +357,7 @@ The variables defined in the :mod:`signal` module are: .. versionadded:: 3.3 -The :mod:`signal` module defines one exception: +The :mod:`!signal` module defines one exception: .. exception:: ItimerError @@ -336,7 +371,7 @@ The :mod:`signal` module defines one exception: alias of :exc:`OSError`. -The :mod:`signal` module defines the following functions: +The :mod:`!signal` module defines the following functions: .. function:: alarm(time) @@ -478,11 +513,11 @@ The :mod:`signal` module defines the following functions: .. versionadded:: 3.3 -.. function:: setitimer(which, seconds, interval=0.0) +.. function:: setitimer(which, seconds, interval=0) Sets given interval timer (one of :const:`signal.ITIMER_REAL`, :const:`signal.ITIMER_VIRTUAL` or :const:`signal.ITIMER_PROF`) specified - by *which* to fire after *seconds* (float is accepted, different from + by *which* to fire after *seconds* (rounded up to microseconds, different from :func:`alarm`) and after that every *interval* seconds (if *interval* is non-zero). The interval timer specified by *which* can be cleared by setting *seconds* to zero. @@ -493,13 +528,18 @@ The :mod:`signal` module defines the following functions: :const:`signal.ITIMER_VIRTUAL` sends :const:`SIGVTALRM`, and :const:`signal.ITIMER_PROF` will deliver :const:`SIGPROF`. - The old values are returned as a tuple: (delay, interval). + The old values are returned as a two-tuple of floats: + (``delay``, ``interval``). Attempting to pass an invalid interval timer will cause an :exc:`ItimerError`. .. availability:: Unix. + .. versionchanged:: 3.15 + Accepts any real numbers as *seconds* and *interval*, not only integers + or floats. + .. function:: getitimer(which) @@ -639,9 +679,8 @@ The :mod:`signal` module defines the following functions: *sigset*. The return value is an object representing the data contained in the - :c:type:`siginfo_t` structure, namely: :attr:`si_signo`, :attr:`si_code`, - :attr:`si_errno`, :attr:`si_pid`, :attr:`si_uid`, :attr:`si_status`, - :attr:`si_band`. + ``siginfo_t`` structure, namely: ``si_signo``, ``si_code``, + ``si_errno``, ``si_pid``, ``si_uid``, ``si_status``, ``si_band``. .. availability:: Unix. @@ -676,6 +715,9 @@ The :mod:`signal` module defines the following functions: by a signal not in *sigset* and the signal handler does not raise an exception (see :pep:`475` for the rationale). + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. _signal-example: diff --git a/Doc/library/site.rst b/Doc/library/site.rst index e98dd83b60e..04895ae4ec5 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -34,7 +34,7 @@ For the head part, it uses ``sys.prefix`` and ``sys.exec_prefix``; empty heads are skipped. For the tail part, it uses the empty string and then :file:`lib/site-packages` (on Windows) or :file:`lib/python{X.Y[t]}/site-packages` (on Unix and macOS). (The -optional suffix "t" indicates the :term:`free threading` build, and is +optional suffix "t" indicates the :term:`free-threaded build`, and is appended if ``"t"`` is present in the :data:`sys.abiflags` constant.) For each of the distinct head-tail combinations, it sees if it refers to an existing @@ -51,11 +51,11 @@ added path for configuration files. .. versionchanged:: 3.14 - :mod:`site` is no longer responsible for updating :data:`sys.prefix` and + :mod:`!site` is no longer responsible for updating :data:`sys.prefix` and :data:`sys.exec_prefix` on :ref:`sys-path-init-virtual-environments`. This is now done during the :ref:`path initialization `. As a result, under :ref:`sys-path-init-virtual-environments`, :data:`sys.prefix` and - :data:`sys.exec_prefix` no longer depend on the :mod:`site` initialization, + :data:`sys.exec_prefix` no longer depend on the :mod:`!site` initialization, and are therefore unaffected by :option:`-S`. .. _site-virtual-environments-configuration: @@ -64,7 +64,8 @@ When running under a :ref:`virtual environment ` and without the :option:`-S` option. -The default behavior is enable tab-completion and to use +The default behavior is to enable tab completion and to use :file:`~/.python_history` as the history save file. To disable it, delete (or override) the :data:`sys.__interactivehook__` attribute in your :mod:`sitecustomize` or :mod:`usercustomize` module or your @@ -270,12 +271,12 @@ Module contents .. _site-commandline: -Command Line Interface +Command-line interface ---------------------- .. program:: site -The :mod:`site` module also provides a way to get the user directories from the +The :mod:`!site` module also provides a way to get the user directories from the command line: .. code-block:: shell-session diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index c5f8516f768..5c97199bc45 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -4,8 +4,6 @@ .. module:: smtplib :synopsis: SMTP protocol client (requires sockets). -.. sectionauthor:: Eric S. Raymond - **Source code:** :source:`Lib/smtplib.py` .. index:: @@ -14,7 +12,7 @@ -------------- -The :mod:`smtplib` module defines an SMTP client session object that can be used +The :mod:`!smtplib` module defines an SMTP client session object that can be used to send mail to any internet machine with an SMTP or ESMTP listener daemon. For details of SMTP and ESMTP operation, consult :rfc:`821` (Simple Mail Transfer Protocol) and :rfc:`1869` (SMTP Service Extensions). @@ -24,10 +22,13 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). .. class:: SMTP(host='', port=0, local_hostname=None[, timeout], source_address=None) An :class:`SMTP` instance encapsulates an SMTP connection. It has methods - that support a full repertoire of SMTP and ESMTP operations. If the optional - *host* and *port* parameters are given, the SMTP :meth:`connect` method is - called with those parameters during initialization. If specified, - *local_hostname* is used as the FQDN of the local host in the HELO/EHLO + that support a full repertoire of SMTP and ESMTP operations. + + If the host parameter is set to a truthy value, :meth:`SMTP.connect` is called with + host and port automatically when the object is created; otherwise, :meth:`!connect` must + be called manually. + + If specified, *local_hostname* is used as the FQDN of the local host in the HELO/EHLO command. Otherwise, the local hostname is found using :func:`socket.getfqdn`. If the :meth:`connect` call returns anything other than a success code, an :exc:`SMTPConnectError` is raised. The optional @@ -62,6 +63,10 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). ``smtplib.SMTP.send`` with arguments ``self`` and ``data``, where ``data`` is the bytes about to be sent to the remote host. + .. attribute:: SMTP.default_port + + The default port used for SMTP connections (25). + .. versionchanged:: 3.3 Support for the :keyword:`with` statement was added. @@ -80,15 +85,23 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). An :class:`SMTP_SSL` instance behaves exactly the same as instances of :class:`SMTP`. :class:`SMTP_SSL` should be used for situations where SSL is - required from the beginning of the connection and using :meth:`starttls` is - not appropriate. If *host* is not specified, the local host is used. If - *port* is zero, the standard SMTP-over-SSL port (465) is used. The optional - arguments *local_hostname*, *timeout* and *source_address* have the same + required from the beginning of the connection and using :meth:`SMTP.starttls` is + not appropriate. + + If the host parameter is set to a truthy value, :meth:`SMTP.connect` is called with host + and port automatically when the object is created; otherwise, :meth:`!SMTP.connect` must + be called manually. + + The optional arguments *local_hostname*, *timeout* and *source_address* have the same meaning as they do in the :class:`SMTP` class. *context*, also optional, can contain a :class:`~ssl.SSLContext` and allows configuring various aspects of the secure connection. Please read :ref:`ssl-security` for best practices. + .. attribute:: SMTP_SSL.default_port + + The default port used for SMTP-over-SSL connections (465). + .. versionchanged:: 3.3 *context* was added. @@ -112,7 +125,7 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). The LMTP protocol, which is very similar to ESMTP, is heavily based on the standard SMTP client. It's common to use Unix sockets for LMTP, so our - :meth:`connect` method must support that as well as a regular host:port + :meth:`~SMTP.connect` method must support that as well as a regular host:port server. The optional arguments *local_hostname* and *source_address* have the same meaning as they do in the :class:`SMTP` class. To specify a Unix socket, you must use an absolute path for *host*, starting with a '/'. @@ -147,9 +160,15 @@ A nice selection of exceptions is defined as well: .. exception:: SMTPResponseException Base class for all exceptions that include an SMTP error code. These exceptions - are generated in some instances when the SMTP server returns an error code. The - error code is stored in the :attr:`smtp_code` attribute of the error, and the - :attr:`smtp_error` attribute is set to the error message. + are generated in some instances when the SMTP server returns an error code. + + .. attribute:: smtp_code + + The error code. + + .. attribute:: smtp_error + + The error message. .. exception:: SMTPSenderRefused @@ -161,9 +180,13 @@ A nice selection of exceptions is defined as well: .. exception:: SMTPRecipientsRefused - All recipient addresses refused. The errors for each recipient are accessible - through the attribute :attr:`recipients`, which is a dictionary of exactly the - same sort as :meth:`SMTP.sendmail` returns. + All recipient addresses refused. + + .. attribute:: recipients + + A dictionary of exactly the same sort as returned + by :meth:`SMTP.sendmail` containing the errors for + each recipient. .. exception:: SMTPDataError @@ -213,7 +236,6 @@ SMTP Objects An :class:`SMTP` instance has the following methods: - .. method:: SMTP.set_debuglevel(level) Set the debug output level. A value of 1 or ``True`` for *level* results in @@ -250,6 +272,9 @@ An :class:`SMTP` instance has the following methods: 2-tuple of the response code and message sent by the server in its connection response. + If port is not changed from its default value of 0, the value of the :attr:`default_port` + attribute is used. + .. audit-event:: smtplib.connect self,host,port smtplib.SMTP.connect @@ -327,7 +352,7 @@ An :class:`SMTP` instance has the following methods: :exc:`SMTPException` No suitable authentication method was found. - Each of the authentication methods supported by :mod:`smtplib` are tried in + Each of the authentication methods supported by :mod:`!smtplib` are tried in turn if they are advertised as supported by the server. See :meth:`auth` for a list of supported authentication methods. *initial_response_ok* is passed through to :meth:`auth`. @@ -379,7 +404,7 @@ An :class:`SMTP` instance has the following methods: call the :meth:`login` method, which will try each of the above mechanisms in turn, in the order listed. ``auth`` is exposed to facilitate the implementation of authentication methods not (or not yet) supported - directly by :mod:`smtplib`. + directly by :mod:`!smtplib`. .. versionadded:: 3.5 @@ -417,7 +442,7 @@ An :class:`SMTP` instance has the following methods: .. versionchanged:: 3.4 The method now supports hostname check with - :attr:`SSLContext.check_hostname` and *Server Name Indicator* (see + :attr:`ssl.SSLContext.check_hostname` and *Server Name Indicator* (see :const:`~ssl.HAS_SNI`). .. versionchanged:: 3.5 @@ -431,11 +456,13 @@ An :class:`SMTP` instance has the following methods: Send mail. The required arguments are an :rfc:`822` from-address string, a list of :rfc:`822` to-address strings (a bare string will be treated as a list with 1 address), and a message string. The caller may pass a list of ESMTP options - (such as ``8bitmime``) to be used in ``MAIL FROM`` commands as *mail_options*. + (such as ``"8bitmime"``) to be used in ``MAIL FROM`` commands as *mail_options*. ESMTP options (such as ``DSN`` commands) that should be used with all ``RCPT`` - commands can be passed as *rcpt_options*. (If you need to use different ESMTP + commands can be passed as *rcpt_options*. Each option should be passed as a string + containing the full text of the option, including any potential key + (for instance, ``"NOTIFY=SUCCESS,FAILURE"``). (If you need to use different ESMTP options to different recipients you have to use the low-level methods such as - :meth:`mail`, :meth:`rcpt` and :meth:`data` to send the message.) + :meth:`!mail`, :meth:`!rcpt` and :meth:`!data` to send the message.) .. note:: @@ -467,10 +494,7 @@ An :class:`SMTP` instance has the following methods: This method may raise the following exceptions: :exc:`SMTPRecipientsRefused` - All recipients were refused. Nobody got the mail. The :attr:`recipients` - attribute of the exception object is a dictionary with information about the - refused recipients (like the one returned when at least one recipient was - accepted). + All recipients were refused. Nobody got the mail. :exc:`SMTPHeloError` The server didn't reply properly to the ``HELO`` greeting. @@ -546,6 +570,30 @@ Low-level methods corresponding to the standard SMTP/ESMTP commands ``HELP``, Normally these do not need to be called directly, so they are not documented here. For details, consult the module code. +Additionally, an SMTP instance has the following attributes: + + +.. attribute:: SMTP.helo_resp + + The response to the ``HELO`` command, see :meth:`helo`. + + +.. attribute:: SMTP.ehlo_resp + + The response to the ``EHLO`` command, see :meth:`ehlo`. + + +.. attribute:: SMTP.does_esmtp + + A boolean value indicating whether the server supports ESMTP, see + :meth:`ehlo`. + + +.. attribute:: SMTP.esmtp_features + + A dictionary of the names of SMTP service extensions supported by the server, + see :meth:`ehlo`. + .. _smtp-example: diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index bc89a3228f0..96bc9e7a0d6 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -39,6 +39,8 @@ is implicit on send operations. A TLS/SSL wrapper for socket objects. +.. _socket-addresses: + Socket families --------------- @@ -83,7 +85,7 @@ created. Socket addresses are represented as follows: - For :const:`AF_INET6` address family, a four-tuple ``(host, port, flowinfo, scope_id)`` is used, where *flowinfo* and *scope_id* represent the ``sin6_flowinfo`` and ``sin6_scope_id`` members in :const:`struct sockaddr_in6` in C. For - :mod:`socket` module methods, *flowinfo* and *scope_id* can be omitted just for + :mod:`!socket` module methods, *flowinfo* and *scope_id* can be omitted just for backward compatibility. Note, however, omission of *scope_id* can cause problems in manipulating scoped IPv6 addresses. @@ -118,10 +120,10 @@ created. Socket addresses are represented as follows: ``'can0'``. The network interface name ``''`` can be used to receive packets from all network interfaces of this family. - - :const:`CAN_ISOTP` protocol require a tuple ``(interface, rx_addr, tx_addr)`` + - :const:`CAN_ISOTP` protocol requires a tuple ``(interface, rx_addr, tx_addr)`` where both additional parameters are unsigned long integer that represent a CAN identifier (standard or extended). - - :const:`CAN_J1939` protocol require a tuple ``(interface, name, pgn, addr)`` + - :const:`CAN_J1939` protocol requires a tuple ``(interface, name, pgn, addr)`` where additional parameters are 64-bit unsigned integer representing the ECU name, a 32-bit unsigned integer representing the Parameter Group Number (PGN), and an 8-bit integer representing the address. @@ -302,7 +304,7 @@ generalization of this based on timeouts is supported through Module contents --------------- -The module :mod:`socket` exports the following elements. +The module :mod:`!socket` exports the following elements. Exceptions @@ -482,6 +484,10 @@ The AF_* and SOCK_* constants are now :class:`AddressFamily` and .. versionchanged:: 3.14 Added support for ``TCP_QUICKACK`` on Windows platforms when available. + .. versionchanged:: 3.15 + ``IPV6_HDRINCL`` was added. + Added support for ``SO_PASSRIGHTS`` on Linux platforms when available. + .. data:: AF_CAN PF_CAN @@ -900,7 +906,7 @@ The following functions all create :ref:`socket objects `. Build a pair of connected socket objects using the given address family, socket type, and protocol number. Address family, socket type, and protocol number are - as for the :func:`~socket.socket` function above. The default family is :const:`AF_UNIX` + as for the :func:`~socket.socket` function. The default family is :const:`AF_UNIX` if defined on the platform; otherwise, the default is :const:`AF_INET`. The newly created sockets are :ref:`non-inheritable `. @@ -996,8 +1002,8 @@ The following functions all create :ref:`socket objects `. Duplicate the file descriptor *fd* (an integer as returned by a file object's :meth:`~io.IOBase.fileno` method) and build a socket object from the result. Address - family, socket type and protocol number are as for the :func:`~socket.socket` function - above. The file descriptor should refer to a socket, but this is not checked --- + family, socket type and protocol number are as for the :func:`~socket.socket` function. + The file descriptor should refer to a socket, but this is not checked --- subsequent operations on the object may fail if the file descriptor is invalid. This function is rarely needed, but can be used to get or set socket options on a socket passed to a program as standard input or output (such as a server @@ -1028,13 +1034,13 @@ The following functions all create :ref:`socket objects `. Other functions ''''''''''''''' -The :mod:`socket` module also offers various network-related services: +The :mod:`!socket` module also offers various network-related services: .. function:: close(fd) Close a socket file descriptor. This is like :func:`os.close`, but for - sockets. On some platforms (most noticeable Windows) :func:`os.close` + sockets. On some platforms (most notably Windows) :func:`os.close` does not work for socket file descriptors. .. versionadded:: 3.7 @@ -1069,10 +1075,16 @@ The :mod:`socket` module also offers various network-related services: a string representing the canonical name of the *host* if :const:`AI_CANONNAME` is part of the *flags* argument; else *canonname* will be empty. *sockaddr* is a tuple describing a socket address, whose - format depends on the returned *family* (a ``(address, port)`` 2-tuple for - :const:`AF_INET`, a ``(address, port, flowinfo, scope_id)`` 4-tuple for - :const:`AF_INET6`), and is meant to be passed to the :meth:`socket.connect` - method. + format depends on the returned *family* and flags Python was compiled with, + and is meant to be passed to the :meth:`socket.connect` method. + + *sockaddr* can be one of the following: + + * a ``(address, port)`` 2-tuple for :const:`AF_INET` + * a ``(address, port, flowinfo, scope_id)`` 4-tuple for :const:`AF_INET6` if + Python was compiled with ``--enable-ipv6`` (the default) + * a 2-tuple containing raw data for :const:`AF_INET6` if Python was + compiled with ``--disable-ipv6`` .. note:: @@ -1407,11 +1419,14 @@ The :mod:`socket` module also offers various network-related services: .. function:: setdefaulttimeout(timeout) - Set the default timeout in seconds (float) for new socket objects. When + Set the default timeout in seconds (real number) for new socket objects. When the socket module is first imported, the default is ``None``. See :meth:`~socket.settimeout` for possible values and their respective meanings. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. + .. function:: sethostname(name) @@ -1552,8 +1567,8 @@ to sockets. .. method:: socket.bind(address) - Bind the socket to *address*. The socket must not already be bound. (The format - of *address* depends on the address family --- see above.) + Bind the socket to *address*. The socket must not already be bound. The format + of *address* depends on the address family --- see :ref:`socket-addresses`. .. audit-event:: socket.bind self,address socket.socket.bind @@ -1586,11 +1601,11 @@ to sockets. .. method:: socket.connect(address) - Connect to a remote socket at *address*. (The format of *address* depends on the - address family --- see above.) + Connect to a remote socket at *address*. The format of *address* depends on the + address family --- see :ref:`socket-addresses`. If the connection is interrupted by a signal, the method waits until the - connection completes, or raise a :exc:`TimeoutError` on timeout, if the + connection completes, or raises a :exc:`TimeoutError` on timeout, if the signal handler doesn't raise an exception and the socket is blocking or has a timeout. For non-blocking sockets, the method raises an :exc:`InterruptedError` exception if the connection is interrupted by a @@ -1662,16 +1677,16 @@ to sockets. .. method:: socket.getpeername() Return the remote address to which the socket is connected. This is useful to - find out the port number of a remote IPv4/v6 socket, for instance. (The format - of the address returned depends on the address family --- see above.) On some - systems this function is not supported. + find out the port number of a remote IPv4/v6 socket, for instance. The format + of the address returned depends on the address family --- see :ref:`socket-addresses`. + On some systems this function is not supported. .. method:: socket.getsockname() Return the socket's own address. This is useful to find out the port number of - an IPv4/v6 socket, for instance. (The format of the address returned depends on - the address family --- see above.) + an IPv4/v6 socket, for instance. The format of the address returned depends on + the address family --- see :ref:`socket-addresses`. .. method:: socket.getsockopt(level, optname[, buflen]) @@ -1783,7 +1798,8 @@ to sockets. where *bytes* is a bytes object representing the data received and *address* is the address of the socket sending the data. See the Unix manual page :manpage:`recv(2)` for the meaning of the optional argument *flags*; it defaults - to zero. (The format of *address* depends on the address family --- see above.) + to zero. The format of *address* depends on the address family --- see + :ref:`socket-addresses`. .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise @@ -1913,8 +1929,8 @@ to sockets. new bytestring. The return value is a pair ``(nbytes, address)`` where *nbytes* is the number of bytes received and *address* is the address of the socket sending the data. See the Unix manual page :manpage:`recv(2)` for the meaning of the - optional argument *flags*; it defaults to zero. (The format of *address* - depends on the address family --- see above.) + optional argument *flags*; it defaults to zero. The format of *address* + depends on the address family --- see :ref:`socket-addresses`. .. method:: socket.recv_into(buffer[, nbytes[, flags]]) @@ -1929,7 +1945,7 @@ to sockets. .. method:: socket.send(bytes[, flags]) Send data to the socket. The socket must be connected to a remote socket. The - optional *flags* argument has the same meaning as for :meth:`recv` above. + optional *flags* argument has the same meaning as for :meth:`recv`. Returns the number of bytes sent. Applications are responsible for checking that all data has been sent; if only some of the data was transmitted, the application needs to attempt delivery of the remaining data. For further @@ -1944,7 +1960,7 @@ to sockets. .. method:: socket.sendall(bytes[, flags]) Send data to the socket. The socket must be connected to a remote socket. The - optional *flags* argument has the same meaning as for :meth:`recv` above. + optional *flags* argument has the same meaning as for :meth:`recv`. Unlike :meth:`send`, this method continues to send data from *bytes* until either all data has been sent or an error occurs. ``None`` is returned on success. On error, an exception is raised, and there is no way to determine how @@ -1965,9 +1981,9 @@ to sockets. Send data to the socket. The socket should not be connected to a remote socket, since the destination socket is specified by *address*. The optional *flags* - argument has the same meaning as for :meth:`recv` above. Return the number of - bytes sent. (The format of *address* depends on the address family --- see - above.) + argument has the same meaning as for :meth:`recv`. Return the number of + bytes sent. The format of *address* depends on the address family --- see + :ref:`socket-addresses`. .. audit-event:: socket.sendto self,address socket.socket.sendto @@ -2073,7 +2089,7 @@ to sockets. .. method:: socket.settimeout(value) Set a timeout on blocking socket operations. The *value* argument can be a - nonnegative floating-point number expressing seconds, or ``None``. + nonnegative real number expressing seconds, or ``None``. If a non-zero value is given, subsequent socket operations will raise a :exc:`timeout` exception if the timeout period *value* has elapsed before the operation has completed. If zero is given, the socket is put in @@ -2085,23 +2101,23 @@ to sockets. The method no longer toggles :const:`SOCK_NONBLOCK` flag on :attr:`socket.type`. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. -.. method:: socket.setsockopt(level, optname, value: int) -.. method:: socket.setsockopt(level, optname, value: buffer) - :noindex: -.. method:: socket.setsockopt(level, optname, None, optlen: int) - :noindex: + +.. method:: socket.setsockopt(level, optname, value: int | Buffer) + socket.setsockopt(level, optname, None, optlen: int) .. index:: pair: module; struct Set the value of the given socket option (see the Unix manual page :manpage:`setsockopt(2)`). The needed symbolic constants are defined in this module (:ref:`!SO_\* etc. `). The value can be an integer, - ``None`` or a :term:`bytes-like object` representing a buffer. In the later + ``None`` or a :term:`bytes-like object` representing a buffer. In the latter case it is up to the caller to ensure that the bytestring contains the proper bits (see the optional built-in module :mod:`struct` for a way to encode C structures as bytestrings). When *value* is set to ``None``, - *optlen* argument is required. It's equivalent to call :c:func:`setsockopt` C + *optlen* argument is required. It's equivalent to calling :c:func:`setsockopt` C function with ``optval=NULL`` and ``optlen=optlen``. .. versionchanged:: 3.5 @@ -2415,7 +2431,7 @@ lead to this error:: This is because the previous execution has left the socket in a ``TIME_WAIT`` state, and can't be immediately reused. -There is a :mod:`socket` flag to set, in order to prevent this, +There is a :mod:`!socket` flag to set, in order to prevent this, :const:`socket.SO_REUSEADDR`:: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 7fb629f7d2f..4c98bb8e3b9 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -8,7 +8,7 @@ -------------- -The :mod:`socketserver` module simplifies the task of writing network servers. +The :mod:`!socketserver` module simplifies the task of writing network servers. .. include:: ../includes/wasm-notavail.rst @@ -24,7 +24,7 @@ There are four basic concrete server classes: :meth:`~BaseServer.server_activate`. The other parameters are passed to the :class:`BaseServer` base class. - .. versionchanged:: next + .. versionchanged:: 3.15 The default queue size is now ``socket.SOMAXCONN`` for :class:`socketserver.TCPServer`. .. class:: UDPServer(server_address, RequestHandlerClass, bind_and_activate=True) @@ -546,7 +546,7 @@ The difference is that the ``readline()`` call in the second handler will call first handler had to use a ``recv()`` loop to accumulate data until a newline itself. If it had just used a single ``recv()`` without the loop it would just have returned what has been received so far from the client. -TCP is stream based: data arrives in the order it was sent, but there no +TCP is stream based: data arrives in the order it was sent, but there is no correlation between client ``send()`` or ``sendall()`` calls and the number of ``recv()`` calls on the server required to receive it. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index d317ead66f9..484260e63dd 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -4,8 +4,6 @@ .. module:: sqlite3 :synopsis: A DB-API 2.0 implementation using SQLite 3.x. -.. sectionauthor:: Gerhard Häring - **Source code:** :source:`Lib/sqlite3/` .. Make sure we always doctest the tutorial with an empty database. @@ -31,7 +29,9 @@ PostgreSQL or Oracle. The :mod:`!sqlite3` module was written by Gerhard Häring. It provides an SQL interface compliant with the DB-API 2.0 specification described by :pep:`249`, and -requires SQLite 3.15.2 or newer. +requires the third-party `SQLite `_ library. + +.. include:: ../includes/optional-module.rst This document includes four main sections: @@ -55,7 +55,7 @@ This document includes four main sections: PEP written by Marc-André Lemburg. -.. We use the following practises for SQL code: +.. We use the following practices for SQL code: - UPPERCASE for keywords - snake_case for schema - single quotes for string literals @@ -289,7 +289,7 @@ Module functions Set it to any combination (using ``|``, bitwise or) of :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES` to enable this. - Column names takes precedence over declared types if both flags are set. + Column names take precedence over declared types if both flags are set. By default (``0``), type detection is disabled. :param isolation_level: @@ -514,7 +514,7 @@ Module constants This constant is only available if Python was compiled with SQLite 3.24.0 or greater. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: threadsafety @@ -620,7 +620,7 @@ Connection objects supplied, this must be a :term:`callable` returning an instance of :class:`Cursor` or its subclasses. - .. method:: blobopen(table, column, row, /, *, readonly=False, name="main") + .. method:: blobopen(table, column, rowid, /, *, readonly=False, name="main") Open a :class:`Blob` handle to an existing :abbr:`BLOB (Binary Large OBject)`. @@ -631,8 +631,8 @@ Connection objects :param str column: The name of the column where the blob is located. - :param str row: - The name of the row where the blob is located. + :param int rowid: + The row id where the blob is located. :param bool readonly: Set to ``True`` if the blob should be opened without write @@ -1611,6 +1611,9 @@ Cursor objects If the *size* parameter is used, then it is best for it to retain the same value from one :meth:`fetchmany` call to the next. + .. versionchanged:: 3.15 + Negative *size* values are rejected by raising :exc:`ValueError`. + .. method:: fetchall() Return all (remaining) rows of a query result as a :class:`list`. @@ -1638,6 +1641,9 @@ Cursor objects Read/write attribute that controls the number of rows returned by :meth:`fetchmany`. The default value is 1 which means a single row would be fetched per call. + .. versionchanged:: 3.15 + Negative values are rejected by raising :exc:`ValueError`. + .. attribute:: connection Read-only attribute that provides the SQLite database :class:`Connection` @@ -2279,7 +2285,7 @@ This section shows recipes for common adapters and converters. .. testcode:: - import datetime + import datetime as dt import sqlite3 def adapt_date_iso(val): @@ -2294,21 +2300,21 @@ This section shows recipes for common adapters and converters. """Adapt datetime.datetime to Unix timestamp.""" return int(val.timestamp()) - sqlite3.register_adapter(datetime.date, adapt_date_iso) - sqlite3.register_adapter(datetime.datetime, adapt_datetime_iso) - sqlite3.register_adapter(datetime.datetime, adapt_datetime_epoch) + sqlite3.register_adapter(dt.date, adapt_date_iso) + sqlite3.register_adapter(dt.datetime, adapt_datetime_iso) + sqlite3.register_adapter(dt.datetime, adapt_datetime_epoch) def convert_date(val): """Convert ISO 8601 date to datetime.date object.""" - return datetime.date.fromisoformat(val.decode()) + return dt.date.fromisoformat(val.decode()) def convert_datetime(val): """Convert ISO 8601 datetime to datetime.datetime object.""" - return datetime.datetime.fromisoformat(val.decode()) + return dt.datetime.fromisoformat(val.decode()) def convert_timestamp(val): """Convert Unix epoch timestamp to datetime.datetime object.""" - return datetime.datetime.fromtimestamp(int(val)) + return dt.datetime.fromtimestamp(int(val)) sqlite3.register_converter("date", convert_date) sqlite3.register_converter("datetime", convert_datetime) @@ -2317,17 +2323,17 @@ This section shows recipes for common adapters and converters. .. testcode:: :hide: - dt = datetime.datetime(2019, 5, 18, 15, 17, 8, 123456) + when = dt.datetime(2019, 5, 18, 15, 17, 8, 123456) - assert adapt_date_iso(dt.date()) == "2019-05-18" - assert convert_date(b"2019-05-18") == dt.date() + assert adapt_date_iso(when.date()) == "2019-05-18" + assert convert_date(b"2019-05-18") == when.date() - assert adapt_datetime_iso(dt) == "2019-05-18T15:17:08.123456" - assert convert_datetime(b"2019-05-18T15:17:08.123456") == dt + assert adapt_datetime_iso(when) == "2019-05-18T15:17:08.123456" + assert convert_datetime(b"2019-05-18T15:17:08.123456") == when # Using current time as fromtimestamp() returns local date/time. # Dropping microseconds as adapt_datetime_epoch truncates fractional second part. - now = datetime.datetime.now().replace(microsecond=0) + now = dt.datetime.now().replace(microsecond=0) current_timestamp = int(now.timestamp()) assert adapt_datetime_epoch(now) == current_timestamp diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 0f2c2b89295..f2c35d1897a 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -4,9 +4,6 @@ .. module:: ssl :synopsis: TLS/SSL wrapper for socket objects -.. moduleauthor:: Bill Janssen -.. sectionauthor:: Bill Janssen - **Source code:** :source:`Lib/ssl.py` .. index:: single: OpenSSL; (use in module ssl) @@ -18,8 +15,9 @@ This module provides access to Transport Layer Security (often known as "Secure Sockets Layer") encryption and peer authentication facilities for network sockets, both client-side and server-side. This module uses the OpenSSL -library. It is available on all modern Unix systems, Windows, macOS, and -probably additional platforms, as long as OpenSSL is installed on that platform. +library. + +.. include:: ../includes/optional-module.rst .. note:: @@ -69,7 +67,7 @@ by SSL sockets created through the :meth:`SSLContext.wrap_socket` method. Use of deprecated constants and functions result in deprecation warnings. -Functions, Constants, and Exceptions +Functions, constants, and exceptions ------------------------------------ @@ -129,7 +127,7 @@ purposes. cafile=None, capath=None, cadata=None) Return a new :class:`SSLContext` object with default settings for - the given *purpose*. The settings are chosen by the :mod:`ssl` module, + the given *purpose*. The settings are chosen by the :mod:`!ssl` module, and usually represent a higher security level than when calling the :class:`SSLContext` constructor directly. @@ -232,7 +230,7 @@ Signature algorithms :meth:`SSLContext.set_client_sigalgs` and :meth:`SSLContext.set_server_sigalgs` methods. - .. versionadded:: next + .. versionadded:: 3.15 Exceptions @@ -354,9 +352,8 @@ Random generation .. function:: RAND_status() Return ``True`` if the SSL pseudo-random number generator has been seeded - with 'enough' randomness, and ``False`` otherwise. You can use - :func:`ssl.RAND_egd` and :func:`ssl.RAND_add` to increase the randomness of - the pseudo-random number generator. + with 'enough' randomness, and ``False`` otherwise. Use :func:`ssl.RAND_add` + to increase the randomness of the pseudo-random number generator. .. function:: RAND_add(bytes, entropy, /) @@ -377,7 +374,7 @@ Certificate handling .. function:: cert_time_to_seconds(cert_time) - Return the time in seconds since the Epoch, given the ``cert_time`` + Return the time in seconds since the epoch, given the ``cert_time`` string representing the "notBefore" or "notAfter" date from a certificate in ``"%b %d %H:%M:%S %Y %Z"`` strptime format (C locale). @@ -387,12 +384,12 @@ Certificate handling .. doctest:: newcontext >>> import ssl + >>> import datetime as dt >>> timestamp = ssl.cert_time_to_seconds("Jan 5 09:34:43 2018 GMT") >>> timestamp # doctest: +SKIP 1515144883 - >>> from datetime import datetime - >>> print(datetime.utcfromtimestamp(timestamp)) # doctest: +SKIP - 2018-01-05 09:34:43 + >>> print(dt.datetime.fromtimestamp(timestamp, dt.UTC)) # doctest: +SKIP + 2018-01-05 09:34:43+00:00 "notBefore" or "notAfter" dates must use GMT (:rfc:`5280`). @@ -959,7 +956,7 @@ Constants Whether the OpenSSL library has built-in support for External PSKs in TLS 1.3 as described in :rfc:`9258`. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: HAS_PHA @@ -1075,7 +1072,7 @@ Constants :attr:`TLSVersion.TLSv1_3` are deprecated. -SSL Sockets +SSL sockets ----------- .. class:: SSLSocket(socket.socket) @@ -1134,7 +1131,7 @@ SSL Sockets functions support reading and writing of data larger than 2 GB. Writing zero-length data no longer fails with a protocol violation error. - .. versionchanged:: next + .. versionchanged:: 3.15 Python now uses ``SSL_sendfile`` internally when possible. The function sends a file more efficiently because it performs TLS encryption in the kernel to avoid additional context switches. @@ -1318,7 +1315,7 @@ SSL sockets also have the following additional methods and attributes: Return the group used for doing key agreement on this connection. If no connection has been established, returns ``None``. - .. versionadded:: next + .. versionadded:: 3.15 .. method:: SSLSocket.client_sigalg() @@ -1326,7 +1323,7 @@ SSL sockets also have the following additional methods and attributes: authentication on this connection, or ``None`` if no connection has been established or client authentication didn't occur. - .. versionadded:: next + .. versionadded:: 3.15 .. method:: SSLSocket.server_sigalg() @@ -1334,7 +1331,7 @@ SSL sockets also have the following additional methods and attributes: handshake on this connection, or ``None`` if no connection has been established or the cipher suite has no signature. - .. versionadded:: next + .. versionadded:: 3.15 .. method:: SSLSocket.compression() @@ -1465,7 +1462,7 @@ SSL sockets also have the following additional methods and attributes: .. versionadded:: 3.6 -SSL Contexts +SSL contexts ------------ .. versionadded:: 3.2 @@ -1510,7 +1507,7 @@ to speed up repeated connections from the same clients. TLS 1.3. .. seealso:: - :func:`create_default_context` lets the :mod:`ssl` module choose + :func:`create_default_context` lets the :mod:`!ssl` module choose security settings for a given purpose. .. versionchanged:: 3.6 @@ -1710,7 +1707,7 @@ to speed up repeated connections from the same clients. :const:`True` this method will also return any associated aliases such as the ECDH curve names supported in older versions of OpenSSL. - .. versionadded:: next + .. versionadded:: 3.15 .. method:: SSLContext.set_default_verify_paths() @@ -1749,7 +1746,7 @@ to speed up repeated connections from the same clients. When connected, the :meth:`SSLSocket.cipher` method of SSL sockets will return details about the negotiated cipher. - .. versionadded:: next + .. versionadded:: 3.15 .. method:: SSLContext.set_groups(groups, /) @@ -1762,7 +1759,7 @@ to speed up repeated connections from the same clients. When connected, the :meth:`SSLSocket.group` method of SSL sockets will return the group used for key agreement on that connection. - .. versionadded:: next + .. versionadded:: 3.15 .. method:: SSLContext.set_client_sigalgs(sigalgs, /) @@ -1777,7 +1774,7 @@ to speed up repeated connections from the same clients. sockets will return the signature algorithm used for performing certificate-based client authentication on that connection. - .. versionadded:: next + .. versionadded:: 3.15 .. method:: SSLContext.set_server_sigalgs(sigalgs, /) @@ -1791,7 +1788,7 @@ to speed up repeated connections from the same clients. sockets will return the signature algorithm used by the server to complete the TLS handshake on that connection. - .. versionadded:: next + .. versionadded:: 3.15 .. method:: SSLContext.set_alpn_protocols(alpn_protocols) @@ -2656,7 +2653,7 @@ thus several things you need to be aware of: as well. -Memory BIO Support +Memory BIO support ------------------ .. versionadded:: 3.5 @@ -2959,16 +2956,16 @@ of TLS/SSL. Some new TLS 1.3 features are not yet available. Steve Kent :rfc:`RFC 4086: Randomness Requirements for Security <4086>` - Donald E., Jeffrey I. Schiller + Donald E. Eastlake, Jeffrey I. Schiller, Steve Crocker :rfc:`RFC 5280: Internet X.509 Public Key Infrastructure Certificate and Certificate Revocation List (CRL) Profile <5280>` - D. Cooper + David Cooper et al. :rfc:`RFC 5246: The Transport Layer Security (TLS) Protocol Version 1.2 <5246>` - T. Dierks et. al. + Tim Dierks and Eric Rescorla. :rfc:`RFC 6066: Transport Layer Security (TLS) Extensions <6066>` - D. Eastlake + Donald E. Eastlake `IANA TLS: Transport Layer Security (TLS) Parameters `_ IANA diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index 8434b2e8c75..5c5f1858ba4 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -5,13 +5,11 @@ :synopsis: Utilities for interpreting the results of os.stat(), os.lstat() and os.fstat(). -.. sectionauthor:: Skip Montanaro - **Source code:** :source:`Lib/stat.py` -------------- -The :mod:`stat` module defines constants and functions for interpreting the +The :mod:`!stat` module defines constants and functions for interpreting the results of :func:`os.stat`, :func:`os.fstat` and :func:`os.lstat` (if they exist). For complete details about the :c:func:`stat`, :c:func:`!fstat` and :c:func:`!lstat` calls, consult the documentation for your system. @@ -19,7 +17,7 @@ exist). For complete details about the :c:func:`stat`, :c:func:`!fstat` and .. versionchanged:: 3.4 The stat module is backed by a C implementation. -The :mod:`stat` module defines the following functions to test for specific file +The :mod:`!stat` module defines the following functions to test for specific file types: @@ -493,3 +491,22 @@ constants, but are not an exhaustive list. IO_REPARSE_TAG_APPEXECLINK .. versionadded:: 3.8 + +On Linux, the following file attribute constants are available for use when +testing bits in the :attr:`~os.statx_result.stx_attributes` and +:attr:`~os.statx_result.stx_attributes_mask` members returned by +:func:`os.statx`. See the :manpage:`statx(2)` man page for more detail on the +meaning of these constants. + +.. data:: STATX_ATTR_COMPRESSED + STATX_ATTR_IMMUTABLE + STATX_ATTR_APPEND + STATX_ATTR_NODUMP + STATX_ATTR_ENCRYPTED + STATX_ATTR_AUTOMOUNT + STATX_ATTR_MOUNT_ROOT + STATX_ATTR_VERITY + STATX_ATTR_DAX + STATX_ATTR_WRITE_ATOMIC + + .. versionadded:: 3.15 diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 614f5b905a4..dba0e26787d 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -4,9 +4,6 @@ .. module:: statistics :synopsis: Mathematical statistics functions -.. moduleauthor:: Steven D'Aprano -.. sectionauthor:: Steven D'Aprano - .. versionadded:: 3.4 **Source code:** :source:`Lib/statistics.py` @@ -716,7 +713,7 @@ However, for reading convenience, most of the examples show sorted sequences. .. function:: covariance(x, y, /) - Return the sample covariance of two inputs *x* and *y*. Covariance + Return the sample covariance of two sequence inputs *x* and *y*. Covariance is a measure of the joint variability of two inputs. Both inputs must be of the same length (no less than two), otherwise @@ -742,7 +739,7 @@ However, for reading convenience, most of the examples show sorted sequences. Return the `Pearson's correlation coefficient `_ - for two inputs. Pearson's correlation coefficient *r* takes values + for two sequence inputs. Pearson's correlation coefficient *r* takes values between -1 and +1. It measures the strength and direction of a linear relationship. @@ -805,7 +802,7 @@ However, for reading convenience, most of the examples show sorted sequences. (it is equal to the difference between predicted and actual values of the dependent variable). - Both inputs must be of the same length (no less than two), and + Both inputs must be sequences of the same length (no less than two), and the independent variable *x* cannot be constant; otherwise a :exc:`StatisticsError` is raised. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index b30b7fef908..3d943566be3 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -46,8 +46,10 @@ Any object can be tested for truth value, for use in an :keyword:`if` or By default, an object is considered true unless its class defines either a :meth:`~object.__bool__` method that returns ``False`` or a :meth:`~object.__len__` method that -returns zero, when called with the object. [1]_ Here are most of the built-in -objects considered false: +returns zero, when called with the object. [1]_ If one of the methods raises an +exception when called, the exception is propagated and the object does +not have a truth value (for example, :data:`NotImplemented`). +Here are most of the built-in objects considered false: .. index:: single: None (Built-in object) @@ -164,7 +166,7 @@ This table summarizes the comparison operations: pair: object; numeric pair: objects; comparing -Objects of different types, except different numeric types, never compare equal. +Unless stated otherwise, objects of different types never compare equal. The ``==`` operator is always defined but for some object types (for example, class objects) is equivalent to :keyword:`is`. The ``<``, ``<=``, ``>`` and ``>=`` operators are only defined where they make sense; for example, they raise a @@ -263,9 +265,17 @@ The constructors :func:`int`, :func:`float`, and pair: operator; % (percent) pair: operator; ** +.. _stdtypes-mixed-arithmetic: + Python fully supports mixed arithmetic: when a binary arithmetic operator has -operands of different numeric types, the operand with the "narrower" type is -widened to that of the other, where integer is narrower than floating point. +operands of different built-in numeric types, the operand with the "narrower" +type is widened to that of the other: + +* If both arguments are complex numbers, no conversion is performed; +* if either argument is a complex or a floating-point number, the other is + converted to a floating-point number; +* otherwise, both must be integers and no conversion is necessary. + Arithmetic with complex and real operands is defined by the usual mathematical formula, for example:: @@ -1091,11 +1101,14 @@ Notes: still ``0``. (4) - The slice of *s* from *i* to *j* is defined as the sequence of items with index - *k* such that ``i <= k < j``. If *i* or *j* is greater than ``len(s)``, use - ``len(s)``. If *i* is omitted or ``None``, use ``0``. If *j* is omitted or - ``None``, use ``len(s)``. If *i* is greater than or equal to *j*, the slice is - empty. + The slice of *s* from *i* to *j* is defined as the sequence of items with + index *k* such that ``i <= k < j``. + + * If *i* is omitted or ``None``, use ``0``. + * If *j* is omitted or ``None``, use ``len(s)``. + * If *i* or *j* is less than ``-len(s)``, use ``0``. + * If *i* or *j* is greater than ``len(s)``, use ``len(s)``. + * If *i* is greater than or equal to *j*, the slice is empty. (5) The slice of *s* from *i* to *j* with step *k* is defined as the sequence of @@ -1150,13 +1163,13 @@ Sequence types also support the following methods: Return the total number of occurrences of *value* in *sequence*. -.. method:: list.index(value[, start[, stop]) - range.index(value[, start[, stop]) - tuple.index(value[, start[, stop]) +.. method:: list.index(value[, start[, stop]]) + range.index(value[, start[, stop]]) + tuple.index(value[, start[, stop]]) :no-contents-entry: :no-index-entry: :no-typesetting: -.. method:: sequence.index(value[, start[, stop]) +.. method:: sequence.index(value[, start[, stop]]) Return the index of the first occurrence of *value* in *sequence*. @@ -1273,7 +1286,7 @@ Mutable sequence types also support the following methods: :no-typesetting: .. method:: sequence.append(value, /) - Append *value* to the end of the sequence + Append *value* to the end of the sequence. This is equivalent to writing ``seq[len(seq):len(seq)] = [value]``. .. method:: bytearray.clear() @@ -1436,6 +1449,11 @@ application). list appear empty for the duration, and raises :exc:`ValueError` if it can detect that the list has been mutated during a sort. +.. seealso:: + + For detailed information on thread-safety guarantees for :class:`list` + objects, see :ref:`thread-safety-list`. + .. _typesseq-tuple: @@ -1842,10 +1860,18 @@ expression support in the :mod:`re` module). lowercase letter ``'ß'`` is equivalent to ``"ss"``. Since it is already lowercase, :meth:`lower` would do nothing to ``'ß'``; :meth:`casefold` converts it to ``"ss"``. + For example: - The casefolding algorithm is - `described in section 3.13 'Default Case Folding' of the Unicode Standard - `__. + .. doctest:: + + >>> 'straße'.lower() + 'straße' + >>> 'straße'.casefold() + 'strasse' + + The casefolding algorithm is `described in section 3.13.3 'Default Case + Folding' of the Unicode Standard + `__. .. versionadded:: 3.3 @@ -1994,10 +2020,16 @@ expression support in the :mod:`re` module). ``{}``. Each replacement field contains either the numeric index of a positional argument, or the name of a keyword argument. Returns a copy of the string where each replacement field is replaced with the string value of - the corresponding argument. + the corresponding argument. For example: + + .. doctest:: >>> "The sum of 1 + 2 is {0}".format(1+2) 'The sum of 1 + 2 is 3' + >>> "The sum of {a} + {b} is {answer}".format(answer=1+2, a=1, b=2) + 'The sum of 1 + 2 is 3' + >>> "{1} expects the {0} Inquisition!".format("Spanish", "Nobody") + 'Nobody expects the Spanish Inquisition!' See :ref:`formatstrings` for a description of the various formatting options that can be specified in format strings. @@ -2037,7 +2069,20 @@ expression support in the :mod:`re` module). .. method:: str.index(sub[, start[, end]]) Like :meth:`~str.find`, but raise :exc:`ValueError` when the substring is - not found. + not found. For example: + + .. doctest:: + + >>> 'spam, spam, spam'.index('spam') + 0 + >>> 'spam, spam, spam'.index('eggs') + Traceback (most recent call last): + File "", line 1, in + 'spam, spam, spam'.index('eggs') + ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ + ValueError: substring not found + + See also :meth:`rindex`. .. method:: str.isalnum() @@ -2045,7 +2090,18 @@ expression support in the :mod:`re` module). Return ``True`` if all characters in the string are alphanumeric and there is at least one character, ``False`` otherwise. A character ``c`` is alphanumeric if one of the following returns ``True``: ``c.isalpha()``, ``c.isdecimal()``, - ``c.isdigit()``, or ``c.isnumeric()``. + ``c.isdigit()``, or ``c.isnumeric()``. For example: + + .. doctest:: + + >>> 'abc123'.isalnum() + True + >>> 'abc123!@#'.isalnum() + False + >>> ''.isalnum() + False + >>> ' '.isalnum() + False .. method:: str.isalpha() @@ -2056,14 +2112,33 @@ expression support in the :mod:`re` module). property being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is different from the `Alphabetic property defined in the section 4.10 'Letters, Alphabetic, and Ideographic' of the Unicode Standard - `_. + `__. + For example: + + .. doctest:: + + >>> 'Letters and spaces'.isalpha() + False + >>> 'LettersOnly'.isalpha() + True + >>> 'µ'.isalpha() # non-ASCII characters can be considered alphabetical too + True + + See :ref:`unicode-properties`. .. method:: str.isascii() Return ``True`` if the string is empty or all characters in the string are ASCII, ``False`` otherwise. - ASCII characters have code points in the range U+0000-U+007F. + ASCII characters have code points in the range U+0000-U+007F. For example: + + .. doctest:: + + >>> 'ASCII characters'.isascii() + True + >>> 'µ'.isascii() + False .. versionadded:: 3.7 @@ -2073,9 +2148,18 @@ expression support in the :mod:`re` module). Return ``True`` if all characters in the string are decimal characters and there is at least one character, ``False`` otherwise. Decimal characters are those that can be used to form - numbers in base 10, e.g. U+0660, ARABIC-INDIC DIGIT + numbers in base 10, such as U+0660, ARABIC-INDIC DIGIT ZERO. Formally a decimal character is a character in the Unicode - General Category "Nd". + General Category "Nd". For example: + + .. doctest:: + + >>> '0123456789'.isdecimal() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isdecimal() # Arabic-Indic digits zero to nine + True + >>> 'alphabetic'.isdecimal() + False .. method:: str.isdigit() @@ -2121,6 +2205,21 @@ expression support in the :mod:`re` module). that have the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION ONE FIFTH. Formally, numeric characters are those with the property value Numeric_Type=Digit, Numeric_Type=Decimal or Numeric_Type=Numeric. + For example: + + .. doctest:: + + >>> '0123456789'.isnumeric() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isnumeric() # Arabic-indic digit zero to nine + True + >>> '⅕'.isnumeric() # Vulgar fraction one fifth + True + >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() + (False, True, True) + + See also :meth:`isdecimal` and :meth:`isdigit`. Numeric characters are + a superset of decimal numbers. .. method:: str.isprintable() @@ -2139,17 +2238,43 @@ expression support in the :mod:`re` module). Nonprintable characters are those in group Separator or Other (Z or C), except the ASCII space. + For example: + + .. doctest:: + + >>> ''.isprintable(), ' '.isprintable() + (True, True) + >>> '\t'.isprintable(), '\n'.isprintable() + (False, False) + + See also :meth:`isspace`. + .. method:: str.isspace() Return ``True`` if there are only whitespace characters in the string and there is at least one character, ``False`` otherwise. + For example: + + .. doctest:: + + >>> ''.isspace() + False + >>> ' '.isspace() + True + >>> '\t\n'.isspace() # TAB and BREAK LINE + True + >>> '\u3000'.isspace() # IDEOGRAPHIC SPACE + True + A character is *whitespace* if in the Unicode character database (see :mod:`unicodedata`), either its general category is ``Zs`` ("Separator, space"), or its bidirectional class is one of ``WS``, ``B``, or ``S``. + See also :meth:`isprintable`. + .. method:: str.istitle() @@ -2157,6 +2282,19 @@ expression support in the :mod:`re` module). character, for example uppercase characters may only follow uncased characters and lowercase characters only cased ones. Return ``False`` otherwise. + For example: + + .. doctest:: + + >>> 'Spam, Spam, Spam'.istitle() + True + >>> 'spam, spam, spam'.istitle() + False + >>> 'SPAM, SPAM, SPAM'.istitle() + False + + See also :meth:`title`. + .. method:: str.isupper() @@ -2181,7 +2319,16 @@ expression support in the :mod:`re` module). Return a string which is the concatenation of the strings in *iterable*. A :exc:`TypeError` will be raised if there are any non-string values in *iterable*, including :class:`bytes` objects. The separator between - elements is the string providing this method. + elements is the string providing this method. For example: + + .. doctest:: + + >>> ', '.join(['spam', 'spam', 'spam']) + 'spam, spam, spam' + >>> '-'.join('Python') + 'P-y-t-h-o-n' + + See also :meth:`split`. .. method:: str.ljust(width, fillchar=' ', /) @@ -2190,15 +2337,33 @@ expression support in the :mod:`re` module). done using the specified *fillchar* (default is an ASCII space). The original string is returned if *width* is less than or equal to ``len(s)``. + For example: + + .. doctest:: + + >>> 'Python'.ljust(10) + 'Python ' + >>> 'Python'.ljust(10, '.') + 'Python....' + >>> 'Monty Python'.ljust(10, '.') + 'Monty Python' + + See also :meth:`rjust`. + .. method:: str.lower() Return a copy of the string with all the cased characters [4]_ converted to - lowercase. + lowercase. For example: - The lowercasing algorithm used is - `described in section 3.13 'Default Case Folding' of the Unicode Standard - `__. + .. doctest:: + + >>> 'Lower Method Example'.lower() + 'lower method example' + + The lowercasing algorithm used is `described in section 3.13.2 'Default Case + Conversion' of the Unicode Standard + `__. .. method:: str.lstrip(chars=None, /) @@ -2237,6 +2402,10 @@ expression support in the :mod:`re` module). the same position in *to*. If there is a third argument, it must be a string, whose characters will be mapped to ``None`` in the result. + .. versionchanged:: 3.15 + + *dict* can now be a :class:`frozendict`. + .. method:: str.partition(sep, /) @@ -2245,12 +2414,27 @@ expression support in the :mod:`re` module). after the separator. If the separator is not found, return a 3-tuple containing the string itself, followed by two empty strings. + For example: + + .. doctest:: + + >>> 'Monty Python'.partition(' ') + ('Monty', ' ', 'Python') + >>> "Monty Python's Flying Circus".partition(' ') + ('Monty', ' ', "Python's Flying Circus") + >>> 'Monty Python'.partition('-') + ('Monty Python', '', '') + + See also :meth:`rpartition`. + .. method:: str.removeprefix(prefix, /) If the string starts with the *prefix* string, return ``string[len(prefix):]``. Otherwise, return a copy of the original - string:: + string: + + .. doctest:: >>> 'TestHook'.removeprefix('Test') 'Hook' @@ -2259,12 +2443,16 @@ expression support in the :mod:`re` module). .. versionadded:: 3.9 + See also :meth:`removesuffix` and :meth:`startswith`. + .. method:: str.removesuffix(suffix, /) If the string ends with the *suffix* string and that *suffix* is not empty, return ``string[:-len(suffix)]``. Otherwise, return a copy of the - original string:: + original string: + + .. doctest:: >>> 'MiscTests'.removesuffix('Tests') 'Misc' @@ -2273,12 +2461,22 @@ expression support in the :mod:`re` module). .. versionadded:: 3.9 + See also :meth:`removeprefix` and :meth:`endswith`. + .. method:: str.replace(old, new, /, count=-1) Return a copy of the string with all occurrences of substring *old* replaced by *new*. If *count* is given, only the first *count* occurrences are replaced. If *count* is not specified or ``-1``, then all occurrences are replaced. + For example: + + .. doctest:: + + >>> 'spam, spam, spam'.replace('spam', 'eggs') + 'eggs, eggs, eggs' + >>> 'spam, spam, spam'.replace('spam', 'eggs', 1) + 'eggs, spam, spam' .. versionchanged:: 3.13 *count* is now supported as a keyword argument. @@ -2289,12 +2487,36 @@ expression support in the :mod:`re` module). Return the highest index in the string where substring *sub* is found, such that *sub* is contained within ``s[start:end]``. Optional arguments *start* and *end* are interpreted as in slice notation. Return ``-1`` on failure. + For example: + + .. doctest:: + + >>> 'spam, spam, spam'.rfind('sp') + 12 + >>> 'spam, spam, spam'.rfind('sp', 0, 10) + 6 + + See also :meth:`find` and :meth:`rindex`. .. method:: str.rindex(sub[, start[, end]]) Like :meth:`rfind` but raises :exc:`ValueError` when the substring *sub* is not found. + For example: + + .. doctest:: + + >>> 'spam, spam, spam'.rindex('spam') + 12 + >>> 'spam, spam, spam'.rindex('eggs') + Traceback (most recent call last): + File "", line 1, in + 'spam, spam, spam'.rindex('eggs') + ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ + ValueError: substring not found + + See also :meth:`index` and :meth:`find`. .. method:: str.rjust(width, fillchar=' ', /) @@ -2303,6 +2525,19 @@ expression support in the :mod:`re` module). done using the specified *fillchar* (default is an ASCII space). The original string is returned if *width* is less than or equal to ``len(s)``. + For example: + + .. doctest:: + + >>> 'Python'.rjust(10) + ' Python' + >>> 'Python'.rjust(10, '.') + '....Python' + >>> 'Monty Python'.rjust(10, '.') + 'Monty Python' + + See also :meth:`ljust` and :meth:`zfill`. + .. method:: str.rpartition(sep, /) @@ -2311,6 +2546,19 @@ expression support in the :mod:`re` module). after the separator. If the separator is not found, return a 3-tuple containing two empty strings, followed by the string itself. + For example: + + .. doctest:: + + >>> 'Monty Python'.rpartition(' ') + ('Monty', ' ', 'Python') + >>> "Monty Python's Flying Circus".rpartition(' ') + ("Monty Python's Flying", ' ', 'Circus') + >>> 'Monty Python'.rpartition('-') + ('', '', 'Monty Python') + + See also :meth:`partition`. + .. method:: str.rsplit(sep=None, maxsplit=-1) @@ -2326,14 +2574,17 @@ expression support in the :mod:`re` module). Return a copy of the string with trailing characters removed. The *chars* argument is a string specifying the set of characters to be removed. If omitted or ``None``, the *chars* argument defaults to removing whitespace. The *chars* - argument is not a suffix; rather, all combinations of its values are stripped:: + argument is not a suffix; rather, all combinations of its values are stripped. + For example: + + .. doctest:: >>> ' spacious '.rstrip() ' spacious' >>> 'mississippi'.rstrip('ipz') 'mississ' - See :meth:`str.removesuffix` for a method that will remove a single suffix + See :meth:`removesuffix` for a method that will remove a single suffix string rather than all of a set of characters. For example:: >>> 'Monty Python'.rstrip(' Python') @@ -2341,6 +2592,9 @@ expression support in the :mod:`re` module). >>> 'Monty Python'.removesuffix(' Python') 'Monty' + See also :meth:`strip`. + + .. method:: str.split(sep=None, maxsplit=-1) Return a list of the words in the string, using *sep* as the delimiter @@ -2395,6 +2649,8 @@ expression support in the :mod:`re` module). >>> " foo ".split(maxsplit=0) ['foo '] + See also :meth:`join`. + .. index:: single: universal newlines; str.splitlines method @@ -2469,6 +2725,19 @@ expression support in the :mod:`re` module). test string beginning at that position. With optional *end*, stop comparing string at that position. + For example: + + .. doctest:: + + >>> 'Python'.startswith('Py') + True + >>> 'a tuple of prefixes'.startswith(('at', 'a')) + True + >>> 'Python is amazing'.startswith('is', 7) + True + + See also :meth:`endswith` and :meth:`removeprefix`. + .. method:: str.strip(chars=None, /) @@ -2476,7 +2745,11 @@ expression support in the :mod:`re` module). The *chars* argument is a string specifying the set of characters to be removed. If omitted or ``None``, the *chars* argument defaults to removing whitespace. The *chars* argument is not a prefix or suffix; rather, all combinations of its - values are stripped:: + values are stripped. + + For example: + + .. doctest:: >>> ' spacious '.strip() 'spacious' @@ -2487,18 +2760,37 @@ expression support in the :mod:`re` module). from the string. Characters are removed from the leading end until reaching a string character that is not contained in the set of characters in *chars*. A similar action takes place on the trailing end. - For example:: + + For example: + + .. doctest:: >>> comment_string = '#....... Section 3.2.1 Issue #32 .......' >>> comment_string.strip('.#! ') 'Section 3.2.1 Issue #32' + See also :meth:`rstrip`. + .. method:: str.swapcase() Return a copy of the string with uppercase characters converted to lowercase and - vice versa. Note that it is not necessarily true that - ``s.swapcase().swapcase() == s``. + vice versa. For example: + + .. doctest:: + + >>> 'Hello World'.swapcase() + 'hELLO wORLD' + + Note that it is not necessarily true that ``s.swapcase().swapcase() == s``. + For example: + + .. doctest:: + + >>> 'straße'.swapcase().swapcase() + 'strasse' + + See also :meth:`str.lower` and :meth:`str.upper`. .. method:: str.title() @@ -2534,6 +2826,8 @@ expression support in the :mod:`re` module). >>> titlecase("they're bill's friends.") "They're Bill's Friends." + See also :meth:`istitle`. + .. method:: str.translate(table, /) @@ -2549,6 +2843,14 @@ expression support in the :mod:`re` module). You can use :meth:`str.maketrans` to create a translation map from character-to-character mappings in different formats. + The following example uses a mapping to replace ``'a'`` with ``'X'``, + ``'b'`` with ``'Y'``, and delete ``'c'``: + + .. doctest:: + + >>> 'abc123'.translate({ord('a'): 'X', ord('b'): 'Y', ord('c'): None}) + 'XY123' + See also the :mod:`codecs` module for a more flexible approach to custom character mappings. @@ -2561,9 +2863,9 @@ expression support in the :mod:`re` module). character(s) is not "Lu" (Letter, uppercase), but e.g. "Lt" (Letter, titlecase). - The uppercasing algorithm used is - `described in section 3.13 'Default Case Folding' of the Unicode Standard - `__. + The uppercasing algorithm used is `described in section 3.13.2 'Default Case + Conversion' of the Unicode Standard + `__. .. method:: str.zfill(width, /) @@ -2574,13 +2876,17 @@ expression support in the :mod:`re` module). than before. The original string is returned if *width* is less than or equal to ``len(s)``. - For example:: + For example: + + .. doctest:: >>> "42".zfill(5) '00042' >>> "-42".zfill(5) '-0042' + See also :meth:`rjust`. + .. index:: single: ! formatted string literal @@ -2596,6 +2902,8 @@ expression support in the :mod:`re` module). single: : (colon); in formatted string literal single: = (equals); for help in debugging using string literals +.. _stdtypes-fstrings: + Formatted String Literals (f-strings) ------------------------------------- @@ -2604,123 +2912,147 @@ Formatted String Literals (f-strings) The :keyword:`await` and :keyword:`async for` can be used in expressions within f-strings. .. versionchanged:: 3.8 - Added the debugging operator (``=``) + Added the debug specifier (``=``) .. versionchanged:: 3.12 Many restrictions on expressions within f-strings have been removed. Notably, nested strings, comments, and backslashes are now permitted. An :dfn:`f-string` (formally a :dfn:`formatted string literal`) is a string literal that is prefixed with ``f`` or ``F``. -This type of string literal allows embedding arbitrary Python expressions -within *replacement fields*, which are delimited by curly brackets (``{}``). -These expressions are evaluated at runtime, similarly to :meth:`str.format`, -and are converted into regular :class:`str` objects. -For example: +This type of string literal allows embedding the results of arbitrary Python +expressions within *replacement fields*, which are delimited by curly +brackets (``{}``). +Each replacement field must contain an expression, optionally followed by: -.. doctest:: +* a *debug specifier* -- an equal sign (``=``); +* a *conversion specifier* -- ``!s``, ``!r`` or ``!a``; and/or +* a *format specifier* prefixed with a colon (``:``). - >>> who = 'nobody' - >>> nationality = 'Spanish' - >>> f'{who.title()} expects the {nationality} Inquisition!' - 'Nobody expects the Spanish Inquisition!' +See the :ref:`Lexical Analysis section on f-strings ` for details +on the syntax of these fields. -It is also possible to use a multi line f-string: +Debug specifier +^^^^^^^^^^^^^^^ -.. doctest:: +.. versionadded:: 3.8 - >>> f'''This is a string - ... on two lines''' - 'This is a string\non two lines' +If a debug specifier -- an equal sign (``=``) -- appears after the replacement +field expression, the resulting f-string will contain the expression's source, +the equal sign, and the value of the expression. +This is often useful for debugging:: -A single opening curly bracket, ``'{'``, marks a *replacement field* that -can contain any Python expression: + >>> number = 14.3 + >>> f'{number=}' + 'number=14.3' -.. doctest:: +Whitespace before, inside and after the expression, as well as whitespace +after the equal sign, is significant --- it is retained in the result:: - >>> nationality = 'Spanish' - >>> f'The {nationality} Inquisition!' - 'The Spanish Inquisition!' + >>> f'{ number - 4 = }' + ' number - 4 = 10.3' -To include a literal ``{`` or ``}``, use a double bracket: -.. doctest:: +Conversion specifier +^^^^^^^^^^^^^^^^^^^^ - >>> x = 42 - >>> f'{{x}} is {x}' - '{x} is 42' - -Functions can also be used, and :ref:`format specifiers `: - -.. doctest:: - - >>> from math import sqrt - >>> f'√2 \N{ALMOST EQUAL TO} {sqrt(2):.5f}' - '√2 ≈ 1.41421' - -Any non-string expression is converted using :func:`str`, by default: - -.. doctest:: +By default, the value of a replacement field expression is converted to +a string using :func:`str`:: >>> from fractions import Fraction - >>> f'{Fraction(1, 3)}' + >>> one_third = Fraction(1, 3) + >>> f'{one_third}' '1/3' -To use an explicit conversion, use the ``!`` (exclamation mark) operator, -followed by any of the valid formats, which are: +When a debug specifier but no format specifier is used, the default conversion +instead uses :func:`repr`:: -========== ============== -Conversion Meaning -========== ============== -``!a`` :func:`ascii` -``!r`` :func:`repr` -``!s`` :func:`str` -========== ============== + >>> f'{one_third = }' + 'one_third = Fraction(1, 3)' -For example: +The conversion can be specified explicitly using one of these specifiers: -.. doctest:: +* ``!s`` for :func:`str` +* ``!r`` for :func:`repr` +* ``!a`` for :func:`ascii` - >>> from fractions import Fraction - >>> f'{Fraction(1, 3)!s}' +For example:: + + >>> str(one_third) '1/3' - >>> f'{Fraction(1, 3)!r}' + >>> repr(one_third) 'Fraction(1, 3)' - >>> question = '¿Dónde está el Presidente?' - >>> print(f'{question!a}') - '\xbfD\xf3nde est\xe1 el Presidente?' -While debugging it may be helpful to see both the expression and its value, -by using the equals sign (``=``) after the expression. -This preserves spaces within the brackets, and can be used with a converter. -By default, the debugging operator uses the :func:`repr` (``!r``) conversion. -For example: + >>> f'{one_third!s} is {one_third!r}' + '1/3 is Fraction(1, 3)' -.. doctest:: + >>> string = "¡kočka 😸!" + >>> ascii(string) + "'\\xa1ko\\u010dka \\U0001f638!'" + + >>> f'{string = !a}' + "string = '\\xa1ko\\u010dka \\U0001f638!'" + + +Format specifier +^^^^^^^^^^^^^^^^ + +After the expression has been evaluated, and possibly converted using an +explicit conversion specifier, it is formatted using the :func:`format` function. +If the replacement field includes a *format specifier* introduced by a colon +(``:``), the specifier is passed to :func:`!format` as the second argument. +The result of :func:`!format` is then used as the final value for the +replacement field. For example:: >>> from fractions import Fraction - >>> calculation = Fraction(1, 3) - >>> f'{calculation=}' - 'calculation=Fraction(1, 3)' - >>> f'{calculation = }' - 'calculation = Fraction(1, 3)' - >>> f'{calculation = !s}' - 'calculation = 1/3' + >>> one_third = Fraction(1, 3) + >>> f'{one_third:.6f}' + '0.333333' + >>> f'{one_third:_^+10}' + '___+1/3___' + >>> >>> f'{one_third!r:_^20}' + '___Fraction(1, 3)___' + >>> f'{one_third = :~>10}~' + 'one_third = ~~~~~~~1/3~' -Once the output has been evaluated, it can be formatted using a -:ref:`format specifier ` following a colon (``':'``). -After the expression has been evaluated, and possibly converted to a string, -the :meth:`!__format__` method of the result is called with the format specifier, -or the empty string if no format specifier is given. -The formatted result is then used as the final value for the replacement field. -For example: +.. _stdtypes-tstrings: -.. doctest:: +Template String Literals (t-strings) +------------------------------------ - >>> from fractions import Fraction - >>> f'{Fraction(1, 7):.6f}' - '0.142857' - >>> f'{Fraction(1, 7):_^+10}' - '___+1/7___' +An :dfn:`t-string` (formally a :dfn:`template string literal`) is +a string literal that is prefixed with ``t`` or ``T``. + +These strings follow the same syntax and evaluation rules as +:ref:`formatted string literals `, +with for the following differences: + +* Rather than evaluating to a ``str`` object, template string literals evaluate + to a :class:`string.templatelib.Template` object. + +* The :func:`format` protocol is not used. + Instead, the format specifier and conversions (if any) are passed to + a new :class:`~string.templatelib.Interpolation` object that is created + for each evaluated expression. + It is up to code that processes the resulting :class:`~string.templatelib.Template` + object to decide how to handle format specifiers and conversions. + +* Format specifiers containing nested replacement fields are evaluated eagerly, + prior to being passed to the :class:`~string.templatelib.Interpolation` object. + For instance, an interpolation of the form ``{amount:.{precision}f}`` will + evaluate the inner expression ``{precision}`` to determine the value of the + ``format_spec`` attribute. + If ``precision`` were to be ``2``, the resulting format specifier + would be ``'.2f'``. + +* When the equals sign ``'='`` is provided in an interpolation expression, + the text of the expression is appended to the literal string that precedes + the relevant interpolation. + This includes the equals sign and any surrounding whitespace. + The :class:`!Interpolation` instance for the expression will be created as + normal, except that :attr:`~string.templatelib.Interpolation.conversion` will + be set to '``r``' (:func:`repr`) by default. + If an explicit conversion or format specifier are provided, + this will override the default behaviour. .. _old-string-formatting: @@ -2884,6 +3216,10 @@ The conversion types are: | | character in the result. | | +------------+-----------------------------------------------------+-------+ +For floating-point formats, the result should be correctly rounded to a given +precision ``p`` of digits after the decimal point. The rounding mode matches +that of the :func:`round` builtin. + Notes: (1) @@ -3158,6 +3494,30 @@ objects. .. versionadded:: 3.14 + .. method:: take_bytes(n=None, /) + + Remove the first *n* bytes from the bytearray and return them as an immutable + :class:`bytes`. + By default (if *n* is ``None``), return all bytes and clear the bytearray. + + If *n* is negative, index from the end and take the first :func:`len` + plus *n* bytes. If *n* is out of bounds, raise :exc:`IndexError`. + + Taking less than the full length will leave remaining bytes in the + :class:`bytearray`, which requires a copy. If the remaining bytes should be + discarded, use :func:`~bytearray.resize` or :keyword:`del` to truncate + then :func:`~bytearray.take_bytes` without a size. + + .. impl-detail:: + + Taking all bytes is a zero-copy operation. + + .. versionadded:: 3.15 + + See the :ref:`What's New ` entry for + common code patterns which can be optimized with + :meth:`bytearray.take_bytes`. + Since bytearray objects are sequences of integers (akin to a list), for a bytearray object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytearray object of length 1. (This contrasts with text strings, where @@ -3168,6 +3528,11 @@ The representation of bytearray objects uses the bytes literal format ``bytearray([46, 46, 46])``. You can always convert a bytearray object into a list of integers using ``list(b)``. +.. seealso:: + + For detailed information on thread-safety guarantees for :class:`bytearray` + objects, see :ref:`thread-safety-bytearray`. + .. _bytes-methods: @@ -3384,12 +3749,13 @@ arbitrary binary data. The separator to search for may be any :term:`bytes-like object`. -.. method:: bytes.replace(old, new, count=-1, /) - bytearray.replace(old, new, count=-1, /) +.. method:: bytes.replace(old, new, /, count=-1) + bytearray.replace(old, new, /, count=-1) Return a copy of the sequence with all occurrences of subsequence *old* - replaced by *new*. If the optional argument *count* is given, only the - first *count* occurrences are replaced. + replaced by *new*. If *count* is given, only the first *count* occurrences + are replaced. If *count* is not specified or ``-1``, then all occurrences + are replaced. The subsequence to search for and its replacement may be any :term:`bytes-like object`. @@ -3399,6 +3765,9 @@ arbitrary binary data. The bytearray version of this method does *not* operate in place - it always produces a new object, even if no changes were made. + .. versionchanged:: 3.15 + *count* is now supported as a keyword argument. + .. method:: bytes.rfind(sub[, start[, end]]) bytearray.rfind(sub[, start[, end]]) @@ -4215,7 +4584,7 @@ copying. types such as :class:`bytes` and :class:`bytearray`, an element is a single byte, but other types such as :class:`array.array` may have bigger elements. - ``len(view)`` is equal to the length of :class:`~memoryview.tolist`, which + ``len(view)`` is equal to the length of :meth:`~memoryview.tolist`, which is the nested list representation of the view. If ``view.ndim = 1``, this is equal to the number of elements in the view. @@ -4565,7 +4934,7 @@ copying. .. versionadded:: 3.14 - .. method:: index(value, start=0, stop=sys.maxsize, /) + .. method:: index(value, start=0, stop=sys.maxsize, /) Return the index of the first occurrence of *value* (at or after index *start* and before index *stop*). @@ -4694,6 +5063,9 @@ copying. .. versionadded:: 3.3 +For information on the thread safety of :class:`memoryview` objects in +the :term:`free-threaded build`, see :ref:`thread-safety-memoryview`. + .. _types-set: @@ -4716,11 +5088,12 @@ other sequence-like behavior. There are currently two built-in set types, :class:`set` and :class:`frozenset`. The :class:`set` type is mutable --- the contents can be changed using methods -like :meth:`~set.add` and :meth:`~set.remove`. Since it is mutable, it has no -hash value and cannot be used as either a dictionary key or as an element of -another set. The :class:`frozenset` type is immutable and :term:`hashable` --- -its contents cannot be altered after it is created; it can therefore be used as -a dictionary key or as an element of another set. +like :meth:`~set.add` and :meth:`~set.remove`. +Since it is mutable, it has no hash value and cannot be used as +either a dictionary key or as an element of another set. +The :class:`frozenset` type is immutable and :term:`hashable` --- +its contents cannot be altered after it is created; +it can therefore be used as a dictionary key or as an element of another set. Non-empty sets (not frozensets) can be created by placing a comma-separated list of elements within braces, for example: ``{'jack', 'sjoerd'}``, in addition to the @@ -4737,170 +5110,183 @@ The constructors for both classes work the same: objects. If *iterable* is not specified, a new empty set is returned. - Sets can be created by several means: +Sets can be created by several means: - * Use a comma-separated list of elements within braces: ``{'jack', 'sjoerd'}`` - * Use a set comprehension: ``{c for c in 'abracadabra' if c not in 'abc'}`` - * Use the type constructor: ``set()``, ``set('foobar')``, ``set(['a', 'b', 'foo'])`` +* Use a comma-separated list of elements within braces: ``{'jack', 'sjoerd'}`` +* Use a set comprehension: ``{c for c in 'abracadabra' if c not in 'abc'}`` +* Use the type constructor: ``set()``, ``set('foobar')``, ``set(['a', 'b', 'foo'])`` - Instances of :class:`set` and :class:`frozenset` provide the following - operations: +Instances of :class:`set` and :class:`frozenset` provide the following +operations: - .. describe:: len(s) +.. describe:: len(s) - Return the number of elements in set *s* (cardinality of *s*). + Return the number of elements in set *s* (cardinality of *s*). - .. describe:: x in s +.. describe:: x in s - Test *x* for membership in *s*. + Test *x* for membership in *s*. - .. describe:: x not in s +.. describe:: x not in s - Test *x* for non-membership in *s*. + Test *x* for non-membership in *s*. - .. method:: isdisjoint(other, /) +.. method:: frozenset.isdisjoint(other, /) + set.isdisjoint(other, /) - Return ``True`` if the set has no elements in common with *other*. Sets are - disjoint if and only if their intersection is the empty set. + Return ``True`` if the set has no elements in common with *other*. Sets are + disjoint if and only if their intersection is the empty set. - .. method:: issubset(other, /) - set <= other +.. method:: frozenset.issubset(other, /) + set.issubset(other, /) +.. describe:: set <= other - Test whether every element in the set is in *other*. + Test whether every element in the set is in *other*. - .. method:: set < other +.. describe:: set < other - Test whether the set is a proper subset of *other*, that is, - ``set <= other and set != other``. + Test whether the set is a proper subset of *other*, that is, + ``set <= other and set != other``. - .. method:: issuperset(other, /) - set >= other +.. method:: frozenset.issuperset(other, /) + set.issuperset(other, /) +.. describe:: set >= other - Test whether every element in *other* is in the set. + Test whether every element in *other* is in the set. - .. method:: set > other +.. describe:: set > other - Test whether the set is a proper superset of *other*, that is, ``set >= - other and set != other``. + Test whether the set is a proper superset of *other*, that is, ``set >= + other and set != other``. - .. method:: union(*others) - set | other | ... +.. method:: frozenset.union(*others) + set.union(*others) +.. describe:: set | other | ... - Return a new set with elements from the set and all others. + Return a new set with elements from the set and all others. - .. method:: intersection(*others) - set & other & ... +.. method:: frozenset.intersection(*others) + set.intersection(*others) +.. describe:: set & other & ... - Return a new set with elements common to the set and all others. + Return a new set with elements common to the set and all others. - .. method:: difference(*others) - set - other - ... +.. method:: frozenset.difference(*others) + set.difference(*others) +.. describe:: set - other - ... - Return a new set with elements in the set that are not in the others. + Return a new set with elements in the set that are not in the others. - .. method:: symmetric_difference(other, /) - set ^ other +.. method:: frozenset.symmetric_difference(other, /) + set.symmetric_difference(other, /) +.. describe:: set ^ other - Return a new set with elements in either the set or *other* but not both. + Return a new set with elements in either the set or *other* but not both. - .. method:: copy() +.. method:: frozenset.copy() + set.copy() - Return a shallow copy of the set. + Return a shallow copy of the set. - Note, the non-operator versions of :meth:`union`, :meth:`intersection`, - :meth:`difference`, :meth:`symmetric_difference`, :meth:`issubset`, and - :meth:`issuperset` methods will accept any iterable as an argument. In - contrast, their operator based counterparts require their arguments to be - sets. This precludes error-prone constructions like ``set('abc') & 'cbs'`` - in favor of the more readable ``set('abc').intersection('cbs')``. +Note, the non-operator versions of :meth:`~frozenset.union`, +:meth:`~frozenset.intersection`, :meth:`~frozenset.difference`, :meth:`~frozenset.symmetric_difference`, :meth:`~frozenset.issubset`, and +:meth:`~frozenset.issuperset` methods will accept any iterable as an argument. In +contrast, their operator based counterparts require their arguments to be +sets. This precludes error-prone constructions like ``set('abc') & 'cbs'`` +in favor of the more readable ``set('abc').intersection('cbs')``. - Both :class:`set` and :class:`frozenset` support set to set comparisons. Two - sets are equal if and only if every element of each set is contained in the - other (each is a subset of the other). A set is less than another set if and - only if the first set is a proper subset of the second set (is a subset, but - is not equal). A set is greater than another set if and only if the first set - is a proper superset of the second set (is a superset, but is not equal). +Both :class:`set` and :class:`frozenset` support set to set comparisons. Two +sets are equal if and only if every element of each set is contained in the +other (each is a subset of the other). A set is less than another set if and +only if the first set is a proper subset of the second set (is a subset, but +is not equal). A set is greater than another set if and only if the first set +is a proper superset of the second set (is a superset, but is not equal). - Instances of :class:`set` are compared to instances of :class:`frozenset` - based on their members. For example, ``set('abc') == frozenset('abc')`` - returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. +Instances of :class:`set` are compared to instances of :class:`frozenset` +based on their members. For example, ``set('abc') == frozenset('abc')`` +returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. - The subset and equality comparisons do not generalize to a total ordering - function. For example, any two nonempty disjoint sets are not equal and are not - subsets of each other, so *all* of the following return ``False``: ``ab``. +The subset and equality comparisons do not generalize to a total ordering +function. For example, any two nonempty disjoint sets are not equal and are not +subsets of each other, so *all* of the following return ``False``: ``ab``. - Since sets only define partial ordering (subset relationships), the output of - the :meth:`list.sort` method is undefined for lists of sets. +Since sets only define partial ordering (subset relationships), the output of +the :meth:`list.sort` method is undefined for lists of sets. - Set elements, like dictionary keys, must be :term:`hashable`. +Set elements, like dictionary keys, must be :term:`hashable`. - Binary operations that mix :class:`set` instances with :class:`frozenset` - return the type of the first operand. For example: ``frozenset('ab') | - set('bc')`` returns an instance of :class:`frozenset`. +Binary operations that mix :class:`set` instances with :class:`frozenset` +return the type of the first operand. For example: ``frozenset('ab') | +set('bc')`` returns an instance of :class:`frozenset`. - The following table lists operations available for :class:`set` that do not - apply to immutable instances of :class:`frozenset`: +The following table lists operations available for :class:`set` that do not +apply to immutable instances of :class:`frozenset`: - .. method:: update(*others) - set |= other | ... +.. method:: set.update(*others) +.. describe:: set |= other | ... - Update the set, adding elements from all others. + Update the set, adding elements from all others. - .. method:: intersection_update(*others) - set &= other & ... +.. method:: set.intersection_update(*others) +.. describe:: set &= other & ... - Update the set, keeping only elements found in it and all others. + Update the set, keeping only elements found in it and all others. - .. method:: difference_update(*others) - set -= other | ... +.. method:: set.difference_update(*others) +.. describe:: set -= other | ... - Update the set, removing elements found in others. + Update the set, removing elements found in others. - .. method:: symmetric_difference_update(other, /) - set ^= other +.. method:: set.symmetric_difference_update(other, /) +.. describe:: set ^= other - Update the set, keeping only elements found in either set, but not in both. + Update the set, keeping only elements found in either set, but not in both. - .. method:: add(elem, /) +.. method:: set.add(elem, /) - Add element *elem* to the set. + Add element *elem* to the set. - .. method:: remove(elem, /) +.. method:: set.remove(elem, /) - Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is - not contained in the set. + Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is + not contained in the set. - .. method:: discard(elem, /) +.. method:: set.discard(elem, /) - Remove element *elem* from the set if it is present. + Remove element *elem* from the set if it is present. - .. method:: pop() +.. method:: set.pop() - Remove and return an arbitrary element from the set. Raises - :exc:`KeyError` if the set is empty. + Remove and return an arbitrary element from the set. Raises + :exc:`KeyError` if the set is empty. - .. method:: clear() +.. method:: set.clear() - Remove all elements from the set. + Remove all elements from the set. - Note, the non-operator versions of the :meth:`update`, - :meth:`intersection_update`, :meth:`difference_update`, and - :meth:`symmetric_difference_update` methods will accept any iterable as an - argument. +Note, the non-operator versions of the :meth:`~set.update`, +:meth:`~set.intersection_update`, :meth:`~set.difference_update`, and +:meth:`~set.symmetric_difference_update` methods will accept any iterable as an +argument. - Note, the *elem* argument to the :meth:`~object.__contains__`, - :meth:`remove`, and - :meth:`discard` methods may be a set. To support searching for an equivalent - frozenset, a temporary one is created from *elem*. +Note, the *elem* argument to the :meth:`~object.__contains__`, +:meth:`~set.remove`, and +:meth:`~set.discard` methods may be a set. To support searching for an equivalent +frozenset, a temporary one is created from *elem*. + +.. seealso:: + + For detailed information on thread-safety guarantees for :class:`set` + objects, see :ref:`thread-safety-set`. .. _typesmapping: -Mapping Types --- :class:`dict` -=============================== +Mapping types --- :class:`!dict`, :class:`!frozendict` +====================================================== .. index:: pair: object; mapping @@ -4911,8 +5297,9 @@ Mapping Types --- :class:`dict` pair: built-in function; len A :term:`mapping` object maps :term:`hashable` values to arbitrary objects. -Mappings are mutable objects. There is currently only one standard mapping -type, the :dfn:`dictionary`. (For other containers see the built-in +There are currently two standard mapping types, the :dfn:`dictionary` and +:class:`frozendict`. +(For other containers see the built-in :class:`list`, :class:`set`, and :class:`tuple` classes, and the :mod:`collections` module.) @@ -4953,9 +5340,6 @@ can be used interchangeably to index the same dictionary entry. being added is already present, the value from the keyword argument replaces the value from the positional argument. - Providing keyword arguments as in the first example only works for keys that - are valid Python identifiers. Otherwise, any valid keys can be used. - Dictionaries compare equal if and only if they have the same ``(key, value)`` pairs (regardless of ordering). Order comparisons ('<', '<=', '>=', '>') raise :exc:`TypeError`. To illustrate dictionary creation and equality, @@ -5185,9 +5569,14 @@ can be used interchangeably to index the same dictionary entry. Dictionaries are now reversible. + .. seealso:: + :class:`frozendict` and :class:`types.MappingProxyType` can be used to + create a read-only view of a :class:`dict`. + .. seealso:: - :class:`types.MappingProxyType` can be used to create a read-only view - of a :class:`dict`. + + For detailed information on thread-safety guarantees for :class:`dict` + objects, see :ref:`thread-safety-dict`. .. _dict-views: @@ -5296,6 +5685,41 @@ An example of dictionary view usage:: 500 +Frozen dictionaries +------------------- + +.. class:: frozendict(**kwargs) + frozendict(mapping, /, **kwargs) + frozendict(iterable, /, **kwargs) + + Return a new frozen dictionary initialized from an optional positional + argument and a possibly empty set of keyword arguments. + + A :class:`!frozendict` has a similar API to the :class:`dict` API, with the + following differences: + + * :class:`!dict` has more methods than :class:`!frozendict`: + + * :meth:`!__delitem__` + * :meth:`!__setitem__` + * :meth:`~dict.clear` + * :meth:`~dict.pop` + * :meth:`~dict.popitem` + * :meth:`~dict.setdefault` + * :meth:`~dict.update` + + * A :class:`!frozendict` can be hashed with ``hash(frozendict)`` if all keys and + values can be hashed. + + * ``frozendict |= other`` does not modify the :class:`!frozendict` in-place but + creates a new frozen dictionary. + + :class:`!frozendict` is not a :class:`!dict` subclass but inherits directly + from ``object``. + + .. versionadded:: 3.15 + + .. _typecontextmanager: Context Manager Types @@ -5341,9 +5765,11 @@ before the statement body is executed and exited when the statement ends: Returning a true value from this method will cause the :keyword:`with` statement to suppress the exception and continue execution with the statement immediately following the :keyword:`!with` statement. Otherwise the exception continues - propagating after this method has finished executing. Exceptions that occur - during execution of this method will replace any exception that occurred in the - body of the :keyword:`!with` statement. + propagating after this method has finished executing. + + If this method raises an exception while handling an earlier exception from the + :keyword:`with` block, the new exception is raised, and the original exception + is stored in its :attr:`~BaseException.__context__` attribute. The exception passed in should never be reraised explicitly - instead, this method should return a false value to indicate that the method completed @@ -5517,6 +5943,7 @@ list is non-exhaustive. * :class:`list` * :class:`dict` * :class:`set` +* :class:`frozendict` * :class:`frozenset` * :class:`type` * :class:`asyncio.Future` @@ -5544,6 +5971,7 @@ list is non-exhaustive. * :class:`collections.abc.MutableMapping` * :class:`collections.abc.Sequence` * :class:`collections.abc.MutableSequence` +* :class:`collections.abc.ByteString` * :class:`collections.abc.MappingView` * :class:`collections.abc.KeysView` * :class:`collections.abc.ItemsView` @@ -5928,7 +6356,7 @@ It is written as ``None``. The Ellipsis Object ------------------- -This object is commonly used used to indicate that something is omitted. +This object is commonly used to indicate that something is omitted. It supports no special operations. There is exactly one ellipsis object, named :const:`Ellipsis` (a built-in name). ``type(Ellipsis)()`` produces the :const:`Ellipsis` singleton. diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 6336a0ec47b..08ccdfa3f45 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -4,7 +4,7 @@ .. module:: string :synopsis: Common string operations. -**Source code:** :source:`Lib/string.py` +**Source code:** :source:`Lib/string/__init__.py` -------------- @@ -82,12 +82,12 @@ The constants defined in this module are: .. _string-formatting: -Custom String Formatting +Custom string formatting ------------------------ The built-in string class provides the ability to do complex variable substitutions and value formatting via the :meth:`~str.format` method described in -:pep:`3101`. The :class:`Formatter` class in the :mod:`string` module allows +:pep:`3101`. The :class:`Formatter` class in the :mod:`!string` module allows you to create and customize your own string formatting behaviors using the same implementation as the built-in :meth:`~str.format` method. @@ -192,7 +192,7 @@ implementation as the built-in :meth:`~str.format` method. .. _formatstrings: -Format String Syntax +Format string syntax -------------------- The :meth:`str.format` method and the :class:`Formatter` class share the same @@ -304,7 +304,7 @@ See the :ref:`formatexamples` section for some examples. .. _formatspec: -Format Specification Mini-Language +Format specification mini-language ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ "Format specifications" are used within replacement fields contained within a @@ -546,6 +546,9 @@ The available presentation types for :class:`float` and | | :class:`float`, and shows all coefficient digits | | | for :class:`~decimal.Decimal`. If ``p=0``, the decimal | | | point is omitted unless the ``#`` option is used. | + | | | + | | For :class:`float`, the exponent always contains at | + | | least two digits, and is zero if the value is zero. | +---------+----------------------------------------------------------+ | ``'E'`` | Scientific notation. Same as ``'e'`` except it uses | | | an upper case 'E' as the separator character. | @@ -756,8 +759,8 @@ Expressing a percentage:: Using type-specific formatting:: - >>> import datetime - >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58) + >>> import datetime as dt + >>> d = dt.datetime(2010, 7, 4, 12, 15, 58) >>> '{:%Y-%m-%d %H:%M:%S}'.format(d) '2010-07-04 12:15:58' @@ -837,7 +840,7 @@ Template strings support ``$``-based substitutions, using the following rules: Any other appearance of ``$`` in the string will result in a :exc:`ValueError` being raised. -The :mod:`string` module provides a :class:`Template` class that implements +The :mod:`!string` module provides a :class:`Template` class that implements these rules. The methods of :class:`Template` are: diff --git a/Doc/library/string.templatelib.rst b/Doc/library/string.templatelib.rst index 85d65fa9de1..a5b2d796aaf 100644 --- a/Doc/library/string.templatelib.rst +++ b/Doc/library/string.templatelib.rst @@ -24,7 +24,7 @@ Template strings Template strings are a mechanism for custom string processing. They have the full flexibility of Python's :ref:`f-strings`, but return a :class:`Template` instance that gives access -to the static and interpolated (in curly braces) parts of a string +to the static and interpolated (in curly brackets) parts of a string *before* they are combined. To write a t-string, use a ``'t'`` prefix instead of an ``'f'``, like so: @@ -258,13 +258,16 @@ Types .. attribute:: expression :type: str - The text of a valid Python expression, or an empty string. + For interpolations created by t-string literals, :attr:`!expression` + is the expression text found inside the curly brackets (``{`` & ``}``), + including any whitespace, excluding the curly brackets themselves, + and ending before the first ``!``, ``:``, or ``=`` if any is present. + For manually created interpolations, :attr:`!expression` is the arbitrary + string provided when constructing the interpolation instance. - The :attr:`.expression` is the original text of the - interpolation's Python expression, if the interpolation was created - from a t-string literal. Developers creating interpolations manually - should either set this to an empty string or choose a suitable valid - Python expression. + We recommend using valid Python expressions or the empty string for the + ``expression`` field of manually created :class:`!Interpolation` + instances, although this is not enforced at runtime. >>> t'{1 + 2}'.interpolations[0].expression '1 + 2' diff --git a/Doc/library/stringprep.rst b/Doc/library/stringprep.rst index 37d5adf0fa9..325ac9ae7c5 100644 --- a/Doc/library/stringprep.rst +++ b/Doc/library/stringprep.rst @@ -4,9 +4,6 @@ .. module:: stringprep :synopsis: String preparation, as per RFC 3453 -.. moduleauthor:: Martin v. Löwis -.. sectionauthor:: Martin v. Löwis - **Source code:** :source:`Lib/stringprep.py` -------------- @@ -26,14 +23,14 @@ define which tables it uses, and what other optional parts of the ``stringprep`` procedure are part of the profile. One example of a ``stringprep`` profile is ``nameprep``, which is used for internationalized domain names. -The module :mod:`stringprep` only exposes the tables from :rfc:`3454`. As these +The module :mod:`!stringprep` only exposes the tables from :rfc:`3454`. As these tables would be very large to represent as dictionaries or lists, the module uses the Unicode character database internally. The module source code itself was generated using the ``mkstringprep.py`` utility. As a result, these tables are exposed as functions, not as data structures. There are two kinds of tables in the RFC: sets and mappings. For a set, -:mod:`stringprep` provides the "characteristic function", i.e. a function that +:mod:`!stringprep` provides the "characteristic function", i.e. a function that returns ``True`` if the parameter is part of the set. For mappings, it provides the mapping function: given the key, it returns the associated value. Below is a list of all functions available in the module. diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 17fc479fd0c..f504f931f0f 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -36,7 +36,7 @@ and the C layer. responsible for defining byte ordering and padding between elements. See :ref:`struct-alignment` for details. -Several :mod:`struct` functions (and methods of :class:`Struct`) take a *buffer* +Several :mod:`!struct` functions (and methods of :class:`Struct`) take a *buffer* argument. This refers to objects that implement the :ref:`bufferobjects` and provide either a readable or read-writable buffer. The most common types used for that purpose are :class:`bytes` and :class:`bytearray`, but many other types @@ -227,34 +227,34 @@ platform-dependent. +--------+--------------------------+--------------------+----------------+------------+ | ``c`` | :c:expr:`char` | bytes of length 1 | 1 | | +--------+--------------------------+--------------------+----------------+------------+ -| ``b`` | :c:expr:`signed char` | integer | 1 | \(1), \(2) | +| ``b`` | :c:expr:`signed char` | int | 1 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``B`` | :c:expr:`unsigned char` | integer | 1 | \(2) | +| ``B`` | :c:expr:`unsigned char` | int | 1 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ | ``?`` | :c:expr:`_Bool` | bool | 1 | \(1) | +--------+--------------------------+--------------------+----------------+------------+ -| ``h`` | :c:expr:`short` | integer | 2 | \(2) | +| ``h`` | :c:expr:`short` | int | 2 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``H`` | :c:expr:`unsigned short` | integer | 2 | \(2) | +| ``H`` | :c:expr:`unsigned short` | int | 2 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``i`` | :c:expr:`int` | integer | 4 | \(2) | +| ``i`` | :c:expr:`int` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``I`` | :c:expr:`unsigned int` | integer | 4 | \(2) | +| ``I`` | :c:expr:`unsigned int` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``l`` | :c:expr:`long` | integer | 4 | \(2) | +| ``l`` | :c:expr:`long` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``L`` | :c:expr:`unsigned long` | integer | 4 | \(2) | +| ``L`` | :c:expr:`unsigned long` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``q`` | :c:expr:`long long` | integer | 8 | \(2) | +| ``q`` | :c:expr:`long long` | int | 8 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``Q`` | :c:expr:`unsigned long | integer | 8 | \(2) | +| ``Q`` | :c:expr:`unsigned long | int | 8 | \(2) | | | long` | | | | +--------+--------------------------+--------------------+----------------+------------+ -| ``n`` | :c:type:`ssize_t` | integer | | \(3) | +| ``n`` | :c:type:`ssize_t` | int | | \(2), \(3) | +--------+--------------------------+--------------------+----------------+------------+ -| ``N`` | :c:type:`size_t` | integer | | \(3) | +| ``N`` | :c:type:`size_t` | int | | \(2), \(3) | +--------+--------------------------+--------------------+----------------+------------+ -| ``e`` | \(6) | float | 2 | \(4) | +| ``e`` | :c:expr:`_Float16` | float | 2 | \(4), \(6) | +--------+--------------------------+--------------------+----------------+------------+ | ``f`` | :c:expr:`float` | float | 4 | \(4) | +--------+--------------------------+--------------------+----------------+------------+ @@ -268,7 +268,7 @@ platform-dependent. +--------+--------------------------+--------------------+----------------+------------+ | ``p`` | :c:expr:`char[]` | bytes | | \(8) | +--------+--------------------------+--------------------+----------------+------------+ -| ``P`` | :c:expr:`void \*` | integer | | \(5) | +| ``P`` | :c:expr:`void \*` | int | | \(2), \(5) | +--------+--------------------------+--------------------+----------------+------------+ .. versionchanged:: 3.3 @@ -280,6 +280,12 @@ platform-dependent. .. versionchanged:: 3.14 Added support for the ``'F'`` and ``'D'`` formats. +.. seealso:: + + The :mod:`array` and :ref:`ctypes ` modules, + as well as third-party modules like `numpy `__, + use similar -- but slightly different -- type codes. + Notes: @@ -322,7 +328,9 @@ Notes: revision of the `IEEE 754 standard `_. It has a sign bit, a 5-bit exponent and 11-bit precision (with 10 bits explicitly stored), and can represent numbers between approximately ``6.1e-05`` and ``6.5e+04`` - at full precision. This type is not widely supported by C compilers: on a + at full precision. This type is not widely supported by C compilers: + it's available as :c:expr:`_Float16` type, if the compiler supports the Annex H + of the C23 standard. On a typical machine, an unsigned short can be used for storage, but not for math operations. See the Wikipedia page on the `half-precision floating-point format `_ for more information. @@ -334,27 +342,31 @@ Notes: The ``'p'`` format character encodes a "Pascal string", meaning a short variable-length string stored in a *fixed number of bytes*, given by the count. The first byte stored is the length of the string, or 255, whichever is - smaller. The bytes of the string follow. If the string passed in to + smaller. The bytes of the string follow. If the byte string passed in to :func:`pack` is too long (longer than the count minus 1), only the leading - ``count-1`` bytes of the string are stored. If the string is shorter than + ``count-1`` bytes of the string are stored. If the byte string is shorter than ``count-1``, it is padded with null bytes so that exactly count bytes in all are used. Note that for :func:`unpack`, the ``'p'`` format character consumes - ``count`` bytes, but that the string returned can never contain more than 255 + ``count`` bytes, but that the :class:`!bytes` object returned can never contain more than 255 bytes. + When packing, arguments of types :class:`bytes` and :class:`bytearray` + are accepted. (9) For the ``'s'`` format character, the count is interpreted as the length of the - bytes, not a repeat count like for the other format characters; for example, + byte string, not a repeat count like for the other format characters; for example, ``'10s'`` means a single 10-byte string mapping to or from a single Python byte string, while ``'10c'`` means 10 separate one byte character elements (e.g., ``cccccccccc``) mapping to or from ten different Python byte objects. (See :ref:`struct-examples` for a concrete demonstration of the difference.) - If a count is not given, it defaults to 1. For packing, the string is + If a count is not given, it defaults to 1. For packing, the byte string is truncated or padded with null bytes as appropriate to make it fit. For - unpacking, the resulting bytes object always has exactly the specified number - of bytes. As a special case, ``'0s'`` means a single, empty string (while + unpacking, the resulting :class:`!bytes` object always has exactly the specified number + of bytes. As a special case, ``'0s'`` means a single, empty byte string (while ``'0c'`` means 0 characters). + When packing, arguments of types :class:`bytes` and :class:`bytearray` + are accepted. (10) For the ``'F'`` and ``'D'`` format characters, the packed representation uses @@ -479,7 +491,7 @@ at the end, assuming the platform's longs are aligned on 4-byte boundaries:: Applications ------------ -Two main applications for the :mod:`struct` module exist, data +Two main applications for the :mod:`!struct` module exist, data interchange between Python and C code within an application or another application compiled using the same compiler (:ref:`native formats`), and data interchange between applications using agreed upon data layout @@ -571,7 +583,7 @@ below were executed on a 32-bit machine:: Classes ------- -The :mod:`struct` module also defines the following type: +The :mod:`!struct` module also defines the following type: .. class:: Struct(format) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 028a7861f36..fe64daa3291 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -4,21 +4,18 @@ .. module:: subprocess :synopsis: Subprocess management. -.. moduleauthor:: Peter Åstrand -.. sectionauthor:: Peter Åstrand - **Source code:** :source:`Lib/subprocess.py` -------------- -The :mod:`subprocess` module allows you to spawn new processes, connect to their +The :mod:`!subprocess` module allows you to spawn new processes, connect to their input/output/error pipes, and obtain their return codes. This module intends to replace several older modules and functions:: os.system os.spawn* -Information about how the :mod:`subprocess` module can be used to replace these +Information about how the :mod:`!subprocess` module can be used to replace these modules and functions can be found in the following sections. .. seealso:: @@ -27,8 +24,8 @@ modules and functions can be found in the following sections. .. include:: ../includes/wasm-mobile-notavail.rst -Using the :mod:`subprocess` Module ----------------------------------- +Using the :mod:`!subprocess` Module +----------------------------------- The recommended approach to invoking subprocesses is to use the :func:`run` function for all use cases it can handle. For more advanced use cases, the @@ -630,6 +627,12 @@ functions. the value in ``pw_uid`` will be used. If the value is an integer, it will be passed verbatim. (POSIX only) + .. note:: + + Specifying *user* will not drop existing supplementary group memberships! + The caller must also pass ``extra_groups=()`` to reduce the group membership + of the child process for security purposes. + .. availability:: POSIX .. versionadded:: 3.9 @@ -649,7 +652,7 @@ functions. If specified, *env* must provide any variables required for the program to execute. On Windows, in order to run a `side-by-side assembly`_ the - specified *env* **must** include a valid :envvar:`SystemRoot`. + specified *env* **must** include a valid ``%SystemRoot%``. .. _side-by-side assembly: https://en.wikipedia.org/wiki/Side-by-Side_Assembly @@ -803,14 +806,29 @@ Instances of the :class:`Popen` class have the following methods: .. note:: - When the ``timeout`` parameter is not ``None``, then (on POSIX) the - function is implemented using a busy loop (non-blocking call and short - sleeps). Use the :mod:`asyncio` module for an asynchronous wait: see + When ``timeout`` is not ``None`` and the platform supports it, an + efficient event-driven mechanism is used to wait for process termination: + + - Linux >= 5.3 uses :func:`os.pidfd_open` + :func:`select.poll` + - macOS and other BSD variants use :func:`select.kqueue` + + ``KQ_FILTER_PROC`` + ``KQ_NOTE_EXIT`` + - Windows uses ``WaitForSingleObject`` + + If none of these mechanisms are available, the function falls back to a + busy loop (non-blocking call and short sleeps). + + .. note:: + + Use the :mod:`asyncio` module for an asynchronous wait: see :class:`asyncio.create_subprocess_exec`. .. versionchanged:: 3.3 *timeout* was added. + .. versionchanged:: 3.15 + if *timeout* is not ``None``, use efficient event-driven implementation + on Linux >= 5.3 and macOS / BSD. + .. method:: Popen.communicate(input=None, timeout=None) Interact with process: Send data to stdin. Read data from stdout and stderr, @@ -831,7 +849,9 @@ Instances of the :class:`Popen` class have the following methods: If the process does not terminate after *timeout* seconds, a :exc:`TimeoutExpired` exception will be raised. Catching this exception and - retrying communication will not lose any output. + retrying communication will not lose any output. Supplying *input* to a + subsequent post-timeout :meth:`communicate` call is in undefined behavior + and may become an error in the future. The child process is not killed if the timeout expires, so in order to cleanup properly a well-behaved application should kill the child process and @@ -844,6 +864,11 @@ Instances of the :class:`Popen` class have the following methods: proc.kill() outs, errs = proc.communicate() + After a call to :meth:`~Popen.communicate` raises :exc:`TimeoutExpired`, do + not call :meth:`~Popen.wait`. Use an additional :meth:`~Popen.communicate` + call to finish handling pipes and populate the :attr:`~Popen.returncode` + attribute. + .. note:: The data read is buffered in memory, so do not use this method if the data @@ -945,6 +970,11 @@ Reassigning them to new values is unsupported: A negative value ``-N`` indicates that the child was terminated by signal ``N`` (POSIX only). + When ``shell=True``, the return code reflects the exit status of the shell + itself (e.g. ``/bin/sh``), which may map signals to codes such as + ``128+N``. See the documentation of the shell (for example, the Bash + manual's Exit Status) for details. + Windows Popen Helpers --------------------- @@ -1034,7 +1064,7 @@ on Windows. Windows Constants ^^^^^^^^^^^^^^^^^ -The :mod:`subprocess` module exposes the following constants. +The :mod:`!subprocess` module exposes the following constants. .. data:: STD_INPUT_HANDLE @@ -1323,8 +1353,8 @@ calls these functions. .. _subprocess-replacements: -Replacing Older Functions with the :mod:`subprocess` Module ------------------------------------------------------------ +Replacing Older Functions with the :mod:`!subprocess` Module +------------------------------------------------------------ In this section, "a becomes b" means that b can be used as a replacement for a. @@ -1340,7 +1370,7 @@ In this section, "a becomes b" means that b can be used as a replacement for a. :attr:`~CalledProcessError.output` attribute of the raised exception. In the following examples, we assume that the relevant functions have already -been imported from the :mod:`subprocess` module. +been imported from the :mod:`!subprocess` module. Replacing :program:`/bin/sh` shell command substitution @@ -1400,7 +1430,7 @@ Notes: * The :func:`os.system` function ignores SIGINT and SIGQUIT signals while the command is running, but the caller must do this separately when - using the :mod:`subprocess` module. + using the :mod:`!subprocess` module. A more realistic example would look like this:: @@ -1473,7 +1503,7 @@ handling consistency are valid for these functions. Return ``(exitcode, output)`` of executing *cmd* in a shell. - Execute the string *cmd* in a shell with :meth:`Popen.check_output` and + Execute the string *cmd* in a shell with :func:`check_output` and return a 2-tuple ``(exitcode, output)``. *encoding* and *errors* are used to decode output; see the notes on :ref:`frequently-used-arguments` for more details. @@ -1584,7 +1614,7 @@ runtime): Disable use of ``posix_spawn()`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -On Linux, :mod:`subprocess` defaults to using the ``vfork()`` system call +On Linux, :mod:`!subprocess` defaults to using the ``vfork()`` system call internally when it is safe to do so rather than ``fork()``. This greatly improves performance. diff --git a/Doc/library/superseded.rst b/Doc/library/superseded.rst index d120c6acf62..b6b2f28d067 100644 --- a/Doc/library/superseded.rst +++ b/Doc/library/superseded.rst @@ -1,7 +1,7 @@ .. _superseded: ****************** -Superseded Modules +Superseded modules ****************** The modules described in this chapter have been superseded by other modules @@ -24,3 +24,4 @@ currently no modules in this latter category. :maxdepth: 1 getopt.rst + profile.rst diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 54e19af4bd6..52a722608db 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -8,24 +8,26 @@ -------------- -.. moduleauthor:: Jeremy Hylton -.. sectionauthor:: Benjamin Peterson - - Symbol tables are generated by the compiler from AST just before bytecode is generated. The symbol table is responsible for calculating the scope of every -identifier in the code. :mod:`symtable` provides an interface to examine these +identifier in the code. :mod:`!symtable` provides an interface to examine these tables. Generating Symbol Tables ------------------------ -.. function:: symtable(code, filename, compile_type) +.. function:: symtable(code, filename, compile_type, *, module=None) Return the toplevel :class:`SymbolTable` for the Python source *code*. *filename* is the name of the file containing the code. *compile_type* is like the *mode* argument to :func:`compile`. + The optional argument *module* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax warnings + by module name. + + .. versionadded:: 3.15 + Added the *module* parameter. Examining Symbol Tables @@ -174,6 +176,12 @@ Examining Symbol Tables Return a tuple containing names of :term:`free (closure) variables ` in this function. + .. method:: get_cells() + + Return a tuple containing names of :term:`cell variables ` in this table. + + .. versionadded:: 3.15 + .. class:: Class @@ -285,6 +293,12 @@ Examining Symbol Tables Return ``True`` if the symbol is referenced in its block, but not assigned to. + .. method:: is_cell() + + Return ``True`` if the symbol is referenced but not assigned in a nested block. + + .. versionadded:: 3.15 + .. method:: is_free_class() Return *True* if a class-scoped symbol is free from @@ -355,7 +369,7 @@ Command-Line Usage .. versionadded:: 3.13 -The :mod:`symtable` module can be executed as a script from the command line. +The :mod:`!symtable` module can be executed as a script from the command line. .. code-block:: sh diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 0f986aa580b..7cca6f2bcda 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -10,17 +10,17 @@ .. note:: - :mod:`sys.monitoring` is a namespace within the :mod:`sys` module, - not an independent module, so there is no need to - ``import sys.monitoring``, simply ``import sys`` and then use - ``sys.monitoring``. + :mod:`!sys.monitoring` is a namespace within the :mod:`sys` module, + not an independent module, and ``import sys.monitoring`` would fail + with a :exc:`ModuleNotFoundError`. Instead, simply ``import sys`` + and then use ``sys.monitoring``. This namespace provides access to the functions and constants necessary to activate and control event monitoring. As programs execute, events occur that might be of interest to tools that -monitor execution. The :mod:`sys.monitoring` namespace provides means to +monitor execution. The :mod:`!sys.monitoring` namespace provides means to receive callbacks when events of interest occur. The monitoring API consists of three components: @@ -180,8 +180,8 @@ Local events '''''''''''' Local events are associated with normal execution of the program and happen -at clearly defined locations. All local events can be disabled. -The local events are: +at clearly defined locations. All local events can be disabled +per location. The local events are: * :monitoring-event:`PY_START` * :monitoring-event:`PY_RESUME` @@ -205,6 +205,8 @@ Using :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` events will give much better performance as they can be disabled independently. +.. _monitoring-ancillary-events: + Ancillary events '''''''''''''''' @@ -216,14 +218,17 @@ by another event: The :monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events are controlled by the :monitoring-event:`CALL` event. -:monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events will only be seen if the -corresponding :monitoring-event:`CALL` event is being monitored. +:monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events will only be +seen if the corresponding :monitoring-event:`CALL` event is being monitored. + + +.. _monitoring-event-global: Other events '''''''''''' Other events are not necessarily tied to a specific location in the -program and cannot be individually disabled. +program and cannot be individually disabled per location. The other events that can be monitored are: @@ -231,6 +236,12 @@ The other events that can be monitored are: * :monitoring-event:`PY_UNWIND` * :monitoring-event:`RAISE` * :monitoring-event:`EXCEPTION_HANDLED` +* :monitoring-event:`RERAISE` + +.. versionchanged:: 3.15 + Other events can now be turned on and disabled on a per code object + basis. Returning :data:`DISABLE` from a callback disables the event + for the entire code object (for the current tool). The STOP_ITERATION event @@ -244,8 +255,7 @@ raise an exception unless it would be visible to other code. To allow tools to monitor for real exceptions without slowing down generators and coroutines, the :monitoring-event:`STOP_ITERATION` event is provided. -:monitoring-event:`STOP_ITERATION` can be locally disabled, unlike -:monitoring-event:`RAISE`. +:monitoring-event:`STOP_ITERATION` can be locally disabled. Note that the :monitoring-event:`STOP_ITERATION` event and the :monitoring-event:`RAISE` event for a :exc:`StopIteration` exception are @@ -289,12 +299,13 @@ in Python (see :ref:`c-api-monitoring`). .. function:: get_local_events(tool_id: int, code: CodeType, /) -> int - Returns all the local events for *code* + Returns all the :ref:`local events ` for *code* .. function:: set_local_events(tool_id: int, code: CodeType, event_set: int, /) -> None - Activates all the local events for *code* which are set in *event_set*. - Raises a :exc:`ValueError` if *tool_id* is not in use. + Activates all the :ref:`local events ` for *code* + which are set in *event_set*. Raises a :exc:`ValueError` if *tool_id* is not + in use. Disabling events @@ -305,14 +316,19 @@ Disabling events A special value that can be returned from a callback function to disable events for the current code location. -Local events can be disabled for a specific code location by returning -:data:`sys.monitoring.DISABLE` from a callback function. This does not change -which events are set, or any other code locations for the same event. +:ref:`Local events ` can be disabled for a specific code +location by returning :data:`sys.monitoring.DISABLE` from a callback function. +This does not change which events are set, or any other code locations for the +same event. -Disabling events for specific locations is very important for high -performance monitoring. For example, a program can be run under a -debugger with no overhead if the debugger disables all monitoring -except for a few breakpoints. +:ref:`Other events ` can be disabled on a per code +object basis by returning :data:`sys.monitoring.DISABLE` from a callback +function. This disables the event for the entire code object (for the current +tool). + +Disabling events for specific locations is very important for high performance +monitoring. For example, a program can be run under a debugger with no overhead +if the debugger disables all monitoring except for a few breakpoints. .. function:: restart_events() -> None diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 34764a7e4f0..6946eb6eeaa 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -13,7 +13,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. data:: abi_info - .. versionadded:: next + .. versionadded:: 3.15 An object containing information about the ABI of the currently running Python interpreter. @@ -560,7 +560,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only in the range 0--127, and produce undefined results otherwise. Some systems have a convention for assigning specific meanings to specific exit codes, but these are generally underdeveloped; Unix programs generally use 2 for command - line syntax errors and 1 for all other kind of errors. If another type of + line syntax errors and 1 for all other kinds of errors. If another type of object is passed, ``None`` is equivalent to passing zero, and any other object is printed to :data:`stderr` and results in an exit code of 1. In particular, ``sys.exit("some error message")`` is a quick way to exit a @@ -911,6 +911,35 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. versionadded:: 3.11 + +.. function:: get_lazy_imports() + + Returns the current lazy imports mode as a string. + + * ``"normal"``: Only imports explicitly marked with the ``lazy`` keyword + are lazy + * ``"all"``: All top-level imports are potentially lazy + * ``"none"``: All lazy imports are suppressed (even explicitly marked + ones) + + See also :func:`set_lazy_imports` and :pep:`810`. + + .. versionadded:: 3.15 + + +.. function:: get_lazy_imports_filter() + + Returns the current lazy imports filter function, or ``None`` if no + filter is set. + + The filter function is called for every potentially lazy import to + determine whether it should actually be lazy. See + :func:`set_lazy_imports_filter` for details on the filter function + signature. + + .. versionadded:: 3.15 + + .. function:: getrefcount(object) Return the reference count of the *object*. The count returned is generally one @@ -1176,10 +1205,14 @@ always available. Unless explicitly noted otherwise, all variables are read-only The size of the seed key of the hash algorithm + .. attribute:: hash_info.cutoff + + Cutoff for small string DJBX33A optimization in range ``[1, cutoff)``. + .. versionadded:: 3.2 .. versionchanged:: 3.4 - Added *algorithm*, *hash_bits* and *seed_bits* + Added *algorithm*, *hash_bits*, *seed_bits*, and *cutoff*. .. data:: hexversion @@ -1715,6 +1748,63 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. versionadded:: 3.11 + +.. function:: set_lazy_imports(mode) + + Sets the global lazy imports mode. The *mode* parameter must be one of + the following strings: + + * ``"normal"``: Only imports explicitly marked with the ``lazy`` keyword + are lazy + * ``"all"``: All top-level imports become potentially lazy + * ``"none"``: All lazy imports are suppressed (even explicitly marked + ones) + + This function is intended for advanced users who need to control lazy + imports across their entire application. Library developers should + generally not use this function as it affects the runtime execution of + applications. + + In addition to the mode, lazy imports can be controlled via the filter + provided by :func:`set_lazy_imports_filter`. + + See also :func:`get_lazy_imports` and :pep:`810`. + + .. versionadded:: 3.15 + + +.. function:: set_lazy_imports_filter(filter) + + Sets the lazy imports filter callback. The *filter* parameter must be a + callable or ``None`` to clear the filter. + + The filter function is called for every potentially lazy import to + determine whether it should actually be lazy. It must have the following + signature:: + + def filter(importing_module: str, imported_module: str, + fromlist: tuple[str, ...] | None) -> bool + + Where: + + * *importing_module* is the name of the module doing the import + * *imported_module* is the resolved name of the module being imported + (for example, ``lazy from .spam import eggs`` passes + ``package.spam``) + * *fromlist* is the tuple of names being imported (for ``from ... import`` + statements), or ``None`` for regular imports + + The filter should return ``True`` to allow the import to be lazy, or + ``False`` to force an eager import. + + This is an advanced feature intended for specialized users who need + fine-grained control over lazy import behavior. + + See also :func:`get_lazy_imports_filter` and :pep:`810`. + + .. versionadded:: 3.15 + + .. function:: setprofile(profilefunc) .. index:: @@ -1993,6 +2083,9 @@ always available. Unless explicitly noted otherwise, all variables are read-only interpreter is pre-release (alpha, beta, or release candidate) then the local and remote interpreters must be the same exact version. + See :ref:`remote-debugging` for more information about the remote debugging + mechanism. + .. audit-event:: sys.remote_exec pid script_path When the code is executed in the remote process, an @@ -2011,6 +2104,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. availability:: Unix, Windows. .. versionadded:: 3.14 + See :pep:`768` for more details. .. function:: _enablelegacywindowsfsencoding() @@ -2205,7 +2299,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only :func:`sys.unraisablehook` can be overridden to control how unraisable exceptions are handled. - .. versionchanged:: next + .. versionchanged:: 3.15 Exceptions are now printed with colorful text. .. seealso:: @@ -2273,7 +2367,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only The version number used to form registry keys on Windows platforms. This is stored as string resource 1000 in the Python DLL. The value is normally the - major and minor versions of the running Python interpreter. It is provided in the :mod:`sys` + major and minor versions of the running Python interpreter. It is provided in the :mod:`!sys` module for informational purposes; modifying this value has no effect on the registry keys used by Python. diff --git a/Doc/library/sys_path_init.rst b/Doc/library/sys_path_init.rst index a37bb59e7ce..e6c2cddbe84 100644 --- a/Doc/library/sys_path_init.rst +++ b/Doc/library/sys_path_init.rst @@ -57,15 +57,19 @@ otherwise they are set to the same value as :data:`sys.base_prefix` and :data:`sys.base_exec_prefix`, respectively. This is used by :ref:`sys-path-init-virtual-environments`. -Finally, the :mod:`site` module is processed and :file:`site-packages` directories -are added to the module search path. A common way to customize the search path is -to create :mod:`sitecustomize` or :mod:`usercustomize` modules as described in -the :mod:`site` module documentation. +Finally, the :mod:`site` module is processed and :file:`site-packages` +directories are added to the module search path. The :envvar:`PYTHONUSERBASE` +environment variable controls where is searched for user site-packages and the +:envvar:`PYTHONNOUSERSITE` environment variable prevents searching for user +site-packages all together. A common way to customize the search path is to +create :mod:`sitecustomize` or :mod:`usercustomize` modules as described in the +:mod:`site` module documentation. .. note:: - Certain command line options may further affect path calculations. - See :option:`-E`, :option:`-I`, :option:`-s` and :option:`-S` for further details. + The command line options :option:`-E`, :option:`-P`, :option:`-I`, + :option:`-S` and :option:`-s` further affect path calculations, see their + documentation for details. .. versionchanged:: 3.14 @@ -96,11 +100,10 @@ Please refer to :mod:`site`'s .. note:: - There are other ways how "virtual environments" could be implemented, this - documentation refers implementations based on the ``pyvenv.cfg`` mechanism, - such as :mod:`venv`. Most virtual environment implementations follow the - model set by :mod:`venv`, but there may be exotic implementations that - diverge from it. + There are other ways "virtual environments" could be implemented. + This documentation refers to implementations based on the ``pyvenv.cfg`` + mechanism, such as :mod:`venv`, that many virtual environment implementations + follow. _pth files ---------- diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 684d14a74c4..8aa912d99ba 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -4,9 +4,6 @@ .. module:: sysconfig :synopsis: Python's configuration information -.. moduleauthor:: Tarek Ziadé -.. sectionauthor:: Tarek Ziadé - .. versionadded:: 3.2 **Source code:** :source:`Lib/sysconfig` @@ -16,7 +13,7 @@ -------------- -The :mod:`sysconfig` module provides access to Python's configuration +The :mod:`!sysconfig` module provides access to Python's configuration information like the list of installation paths and the configuration variables relevant for the current platform. @@ -28,7 +25,7 @@ A Python distribution contains a :file:`Makefile` and a :file:`pyconfig.h` header file that are necessary to build both the Python binary itself and third-party C extensions compiled using ``setuptools``. -:mod:`sysconfig` puts all variables found in these files in a dictionary that +:mod:`!sysconfig` puts all variables found in these files in a dictionary that can be accessed using :func:`get_config_vars` or :func:`get_config_var`. Notice that on Windows, it's a much smaller set. @@ -68,7 +65,7 @@ Installation paths ------------------ Python uses an installation scheme that differs depending on the platform and on -the installation options. These schemes are stored in :mod:`sysconfig` under +the installation options. These schemes are stored in :mod:`!sysconfig` under unique identifiers based on the value returned by :const:`os.name`. The schemes are used by package installers to determine where to copy files to. @@ -258,12 +255,12 @@ Path Installation directory Installation path functions --------------------------- -:mod:`sysconfig` provides some functions to determine these installation paths. +:mod:`!sysconfig` provides some functions to determine these installation paths. .. function:: get_scheme_names() Return a tuple containing all schemes currently supported in - :mod:`sysconfig`. + :mod:`!sysconfig`. .. function:: get_default_scheme() @@ -285,7 +282,7 @@ Installation path functions *key* must be either ``"prefix"``, ``"home"``, or ``"user"``. The return value is a scheme name listed in :func:`get_scheme_names`. It - can be passed to :mod:`sysconfig` functions that take a *scheme* argument, + can be passed to :mod:`!sysconfig` functions that take a *scheme* argument, such as :func:`get_paths`. .. versionadded:: 3.10 @@ -313,7 +310,7 @@ Installation path functions .. function:: get_path_names() Return a tuple containing all path names currently supported in - :mod:`sysconfig`. + :mod:`!sysconfig`. .. function:: get_path(name, [scheme, [vars, [expand]]]) @@ -323,7 +320,7 @@ Installation path functions *name* has to be a value from the list returned by :func:`get_path_names`. - :mod:`sysconfig` stores installation paths corresponding to each path name, + :mod:`!sysconfig` stores installation paths corresponding to each path name, for each platform, with variables to be expanded. For instance the *stdlib* path for the *nt* scheme is: ``{base}/Lib``. @@ -382,22 +379,22 @@ Other functions Examples of returned values: - - linux-i586 - - linux-alpha (?) + - linux-x86_64 + - linux-aarch64 - solaris-2.6-sun4u - Windows will return one of: + Windows: - win-amd64 (64-bit Windows on AMD64, aka x86_64, Intel64, and EM64T) - win-arm64 (64-bit Windows on ARM64, aka AArch64) - win32 (all others - specifically, sys.platform is returned) - macOS can return: + POSIX based OS: - - macosx-10.6-ppc - - macosx-10.4-ppc64 - - macosx-10.3-i386 - - macosx-10.4-fat + - linux-x86_64 + - macosx-15.5-arm64 + - macosx-26.0-universal2 (macOS on Apple Silicon or Intel) + - android-24-arm64_v8a For other non-POSIX platforms, currently just returns :data:`sys.platform`. @@ -434,7 +431,7 @@ Other functions Command-line usage ------------------ -You can use :mod:`sysconfig` as a script with Python's *-m* option: +You can use :mod:`!sysconfig` as a script with Python's *-m* option: .. code-block:: shell-session diff --git a/Doc/library/syslog.rst b/Doc/library/syslog.rst index 548898a37bc..b6bf0124095 100644 --- a/Doc/library/syslog.rst +++ b/Doc/library/syslog.rst @@ -2,7 +2,6 @@ =============================================== .. module:: syslog - :platform: Unix :synopsis: An interface to the Unix syslog library routines. -------------- diff --git a/Doc/library/tabnanny.rst b/Doc/library/tabnanny.rst index 4f61b3dd761..570cc7fd93b 100644 --- a/Doc/library/tabnanny.rst +++ b/Doc/library/tabnanny.rst @@ -5,11 +5,6 @@ :synopsis: Tool for detecting white space related problems in Python source files in a directory tree. -.. moduleauthor:: Tim Peters -.. sectionauthor:: Peter Funk - -.. rudimentary documentation based on module comments - **Source code:** :source:`Lib/tabnanny.py` -------------- diff --git a/Doc/library/tachyon-flamegraph.png b/Doc/library/tachyon-flamegraph.png new file mode 100644 index 00000000000..a17cd304f8b Binary files /dev/null and b/Doc/library/tachyon-flamegraph.png differ diff --git a/Doc/library/tachyon-gecko-calltree.png b/Doc/library/tachyon-gecko-calltree.png new file mode 100644 index 00000000000..71b096940e8 Binary files /dev/null and b/Doc/library/tachyon-gecko-calltree.png differ diff --git a/Doc/library/tachyon-gecko-flamegraph.png b/Doc/library/tachyon-gecko-flamegraph.png new file mode 100644 index 00000000000..d427ed85ac0 Binary files /dev/null and b/Doc/library/tachyon-gecko-flamegraph.png differ diff --git a/Doc/library/tachyon-gecko-opcodes.png b/Doc/library/tachyon-gecko-opcodes.png new file mode 100644 index 00000000000..9741eb65912 Binary files /dev/null and b/Doc/library/tachyon-gecko-opcodes.png differ diff --git a/Doc/library/tachyon-heatmap-with-opcodes.png b/Doc/library/tachyon-heatmap-with-opcodes.png new file mode 100644 index 00000000000..5ad67d13154 Binary files /dev/null and b/Doc/library/tachyon-heatmap-with-opcodes.png differ diff --git a/Doc/library/tachyon-heatmap.png b/Doc/library/tachyon-heatmap.png new file mode 100644 index 00000000000..47ac1119f4e Binary files /dev/null and b/Doc/library/tachyon-heatmap.png differ diff --git a/Doc/library/tachyon-live-mode-1.gif b/Doc/library/tachyon-live-mode-1.gif new file mode 100644 index 00000000000..2d58e807d6a Binary files /dev/null and b/Doc/library/tachyon-live-mode-1.gif differ diff --git a/Doc/library/tachyon-live-mode-2.gif b/Doc/library/tachyon-live-mode-2.gif new file mode 100644 index 00000000000..bbc2163fe60 Binary files /dev/null and b/Doc/library/tachyon-live-mode-2.gif differ diff --git a/Doc/library/tachyon-pstats.png b/Doc/library/tachyon-pstats.png new file mode 100644 index 00000000000..d0281ade660 Binary files /dev/null and b/Doc/library/tachyon-pstats.png differ diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index acaec5b592b..6f1e01cf5aa 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -4,14 +4,11 @@ .. module:: tarfile :synopsis: Read and write tar-format archive files. -.. moduleauthor:: Lars Gustäbel -.. sectionauthor:: Lars Gustäbel - **Source code:** :source:`Lib/tarfile.py` -------------- -The :mod:`tarfile` module makes it possible to read and write tar +The :mod:`!tarfile` module makes it possible to read and write tar archives, including those using gzip, bz2 and lzma compression. Use the :mod:`zipfile` module to read or write :file:`.zip` files, or the higher-level functions in :ref:`shutil `. @@ -21,6 +18,14 @@ Some facts and figures: * reads and writes :mod:`gzip`, :mod:`bz2`, :mod:`compression.zstd`, and :mod:`lzma` compressed archives if the respective modules are available. + .. + The following paragraph should be similar to ../includes/optional-module.rst + + If any of these :term:`optional modules ` are missing from + your copy of CPython, look for documentation from your distributor (that is, + whoever provided Python to you). + If you are the distributor, see :ref:`optional-module-requirements`. + * read/write support for the POSIX.1-1988 (ustar) format. * read/write support for the GNU tar format including *longname* and *longlink* @@ -137,6 +142,10 @@ Some facts and figures: a Zstandard dictionary used to improve compression of smaller amounts of data. + For modes ``'w:gz'`` and ``'w|gz'``, :func:`tarfile.open` accepts the + keyword argument *mtime* to create a gzip archive header with that mtime. By + default, the mtime is set to the time of creation of the archive. + For special purposes, there is a second format for *mode*: ``'filemode|[compression]'``. :func:`tarfile.open` will return a :class:`TarFile` object that processes its data as a stream of blocks. No random seeking will @@ -198,7 +207,7 @@ Some facts and figures: .. versionchanged:: 3.14 The *preset* keyword argument also works for streams. - .. versionchanged:: next + .. versionchanged:: 3.15 The default compression level was reduced to 6 (down from 9). It is the default level used by most compression tools and a better tradeoff between speed and performance. @@ -212,25 +221,25 @@ Some facts and figures: .. function:: is_tarfile(name) - Return :const:`True` if *name* is a tar archive file, that the :mod:`tarfile` + Return :const:`True` if *name* is a tar archive file, that the :mod:`!tarfile` module can read. *name* may be a :class:`str`, file, or file-like object. .. versionchanged:: 3.9 Support for file and file-like objects. -The :mod:`tarfile` module defines the following exceptions: +The :mod:`!tarfile` module defines the following exceptions: .. exception:: TarError - Base class for all :mod:`tarfile` exceptions. + Base class for all :mod:`!tarfile` exceptions. .. exception:: ReadError Is raised when a tar archive is opened, that either cannot be handled by the - :mod:`tarfile` module or is somehow invalid. + :mod:`!tarfile` module or is somehow invalid. .. exception:: CompressionError @@ -294,7 +303,7 @@ The :mod:`tarfile` module defines the following exceptions: The exception that was raised to reject the replacement member is available as :attr:`!BaseException.__context__`. - .. versionadded:: next + .. versionadded:: 3.15 The following constants are available at the module level: @@ -351,7 +360,7 @@ The following constants are available at the module level: Each of the following constants defines a tar archive format that the -:mod:`tarfile` module is able to create. See section :ref:`tar-formats` for +:mod:`!tarfile` module is able to create. See section :ref:`tar-formats` for details. @@ -1146,7 +1155,7 @@ reused in custom filters: Note that this filter does not block *all* dangerous archive features. See :ref:`tarfile-further-verification` for details. - .. versionchanged:: next + .. versionchanged:: 3.15 Link targets are now normalized. @@ -1281,7 +1290,7 @@ Command-Line Interface .. versionadded:: 3.4 -The :mod:`tarfile` module provides a simple command-line interface to interact +The :mod:`!tarfile` module provides a simple command-line interface to interact with tar archives. If you want to create a new tar archive, specify its name after the :option:`-c` @@ -1442,7 +1451,7 @@ parameter in :meth:`TarFile.add`:: Supported tar formats --------------------- -There are three tar formats that can be created with the :mod:`tarfile` module: +There are three tar formats that can be created with the :mod:`!tarfile` module: * The POSIX.1-1988 ustar format (:const:`USTAR_FORMAT`). It supports filenames up to a length of at best 256 characters and linknames up to 100 characters. @@ -1451,7 +1460,7 @@ There are three tar formats that can be created with the :mod:`tarfile` module: * The GNU tar format (:const:`GNU_FORMAT`). It supports long filenames and linknames, files bigger than 8 GiB and sparse files. It is the de facto - standard on GNU/Linux systems. :mod:`tarfile` fully supports the GNU tar + standard on GNU/Linux systems. :mod:`!tarfile` fully supports the GNU tar extensions for long names, sparse file support is read-only. * The POSIX.1-2001 pax format (:const:`PAX_FORMAT`). It is the most flexible @@ -1496,7 +1505,7 @@ Unfortunately, there is no way to autodetect the encoding of an archive. The pax format was designed to solve this problem. It stores non-ASCII metadata using the universal character encoding *UTF-8*. -The details of character conversion in :mod:`tarfile` are controlled by the +The details of character conversion in :mod:`!tarfile` are controlled by the *encoding* and *errors* keyword arguments of the :class:`TarFile` class. *encoding* defines the character encoding to use for the metadata in the diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index f0a81a093b4..bf9198e175a 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -4,8 +4,6 @@ .. module:: tempfile :synopsis: Generate temporary files and directories. -.. sectionauthor:: Zack Weinberg - **Source code:** :source:`Lib/tempfile.py` .. index:: @@ -225,8 +223,9 @@ The module defines the following user-callable items: properly implements the :const:`os.O_EXCL` flag for :func:`os.open`. The file is readable and writable only by the creating user ID. If the platform uses permission bits to indicate whether a file is executable, - the file is executable by no one. The file descriptor is not inherited - by child processes. + the file is executable by no one. + + The file descriptor is :ref:`not inherited by child processes `. Unlike :func:`TemporaryFile`, the user of :func:`mkstemp` is responsible for deleting the temporary file when done with it. @@ -385,7 +384,7 @@ not surprise other unsuspecting code by changing global API behavior. Examples -------- -Here are some examples of typical usage of the :mod:`tempfile` module:: +Here are some examples of typical usage of the :mod:`!tempfile` module:: >>> import tempfile diff --git a/Doc/library/termios.rst b/Doc/library/termios.rst index 0c6f3059fe7..537dfcedd8c 100644 --- a/Doc/library/termios.rst +++ b/Doc/library/termios.rst @@ -2,7 +2,6 @@ =========================================== .. module:: termios - :platform: Unix :synopsis: POSIX style tty control. .. index:: @@ -38,7 +37,7 @@ The module defines the following functions: items with indices :const:`VMIN` and :const:`VTIME`, which are integers when these fields are defined). The interpretation of the flags and the speeds as well as the indexing in the *cc* array must be done using the symbolic - constants defined in the :mod:`termios` module. + constants defined in the :mod:`!termios` module. .. function:: tcsetattr(fd, when, attributes) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 395cde21ccf..7ae3fabf1ce 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -4,10 +4,8 @@ .. module:: test :synopsis: Regression tests package containing the testing suite for Python. -.. sectionauthor:: Brett Cannon - .. note:: - The :mod:`test` package is meant for internal use by Python only. It is + The :mod:`!test` package is meant for internal use by Python only. It is documented for the benefit of the core developers of Python. Any use of this package outside of Python's standard library is discouraged as code mentioned here can change or be removed without notice between releases of @@ -15,12 +13,12 @@ -------------- -The :mod:`test` package contains all regression tests for Python as well as the +The :mod:`!test` package contains all regression tests for Python as well as the modules :mod:`test.support` and :mod:`test.regrtest`. :mod:`test.support` is used to enhance your tests while :mod:`test.regrtest` drives the testing suite. -Each module in the :mod:`test` package whose name starts with ``test_`` is a +Each module in the :mod:`!test` package whose name starts with ``test_`` is a testing suite for a specific module or feature. All new tests should be written using the :mod:`unittest` or :mod:`doctest` module. Some older tests are written using a "traditional" testing style that compares output printed to @@ -38,8 +36,8 @@ written using a "traditional" testing style that compares output printed to .. _writing-tests: -Writing Unit Tests for the :mod:`test` package ----------------------------------------------- +Writing Unit Tests for the :mod:`!test` package +----------------------------------------------- It is preferred that tests that use the :mod:`unittest` module follow a few guidelines. One is to name the test module by starting it with ``test_`` and end @@ -162,12 +160,12 @@ Running tests using the command-line interface .. module:: test.regrtest :synopsis: Drives the regression test suite. -The :mod:`test` package can be run as a script to drive Python's regression +The :mod:`!test` package can be run as a script to drive Python's regression test suite, thanks to the :option:`-m` option: :program:`python -m test`. Under -the hood, it uses :mod:`test.regrtest`; the call :program:`python -m +the hood, it uses :mod:`!test.regrtest`; the call :program:`python -m test.regrtest` used in previous Python versions still works. Running the script by itself automatically starts running all regression tests in the -:mod:`test` package. It does this by finding all modules in the package whose +:mod:`!test` package. It does this by finding all modules in the package whose name starts with ``test_``, importing them, and executing the function :func:`test_main` if present or loading the tests via unittest.TestLoader.loadTestsFromModule if ``test_main`` does not exist. The @@ -175,14 +173,14 @@ names of tests to execute may also be passed to the script. Specifying a single regression test (:program:`python -m test test_spam`) will minimize output and only print whether the test passed or failed. -Running :mod:`test` directly allows what resources are available for +Running :mod:`!test` directly allows what resources are available for tests to use to be set. You do this by using the ``-u`` command-line option. Specifying ``all`` as the value for the ``-u`` option enables all possible resources: :program:`python -m test -uall`. If all but one resource is desired (a more common case), a comma-separated list of resources that are not desired may be listed after ``all``. The command :program:`python -m test -uall,-audio,-largefile` -will run :mod:`test` with all resources except the ``audio`` and +will run :mod:`!test` with all resources except the ``audio`` and ``largefile`` resources. For a list of all resources and more command-line options, run :program:`python -m test -h`. @@ -197,19 +195,19 @@ regression tests. :ref:`controlled using environment variables `. -:mod:`test.support` --- Utilities for the Python test suite -=========================================================== +:mod:`!test.support` --- Utilities for the Python test suite +============================================================ .. module:: test.support :synopsis: Support for Python's regression test suite. -The :mod:`test.support` module provides support for Python's regression +The :mod:`!test.support` module provides support for Python's regression test suite. .. note:: - :mod:`test.support` is not a public module. It is documented here to help + :mod:`!test.support` is not a public module. It is documented here to help Python developers write tests. The API of this module is subject to change without backwards compatibility concerns between releases. @@ -230,7 +228,7 @@ This module defines the following exceptions: function. -The :mod:`test.support` module defines the following constants: +The :mod:`!test.support` module defines the following constants: .. data:: verbose @@ -363,7 +361,7 @@ The :mod:`test.support` module defines the following constants: .. data:: TEST_SUPPORT_DIR - Set to the top level directory that contains :mod:`test.support`. + Set to the top level directory that contains :mod:`!test.support`. .. data:: TEST_HOME_DIR @@ -438,7 +436,7 @@ The :mod:`test.support` module defines the following constants: Used to test mixed type comparison. -The :mod:`test.support` module defines the following functions: +The :mod:`!test.support` module defines the following functions: .. function:: busy_retry(timeout, err_msg=None, /, *, error=True) @@ -492,6 +490,12 @@ The :mod:`test.support` module defines the following functions: tests. +.. function:: get_resource_value(resource) + + Return the value specified for *resource* (as :samp:`-u {resource}={value}`). + Return ``None`` if *resource* is disabled or no value is specified. + + .. function:: python_is_optimized() Return ``True`` if Python was not built with ``-O0`` or ``-Og``. @@ -1037,7 +1041,7 @@ The :mod:`test.support` module defines the following functions: .. versionadded:: 3.11 -The :mod:`test.support` module defines the following classes: +The :mod:`!test.support` module defines the following classes: .. class:: SuppressCrashReport() @@ -1083,14 +1087,14 @@ The :mod:`test.support` module defines the following classes: Try to match a single stored value (*dv*) with a supplied value (*v*). -:mod:`test.support.socket_helper` --- Utilities for socket tests -================================================================ +:mod:`!test.support.socket_helper` --- Utilities for socket tests +================================================================= .. module:: test.support.socket_helper :synopsis: Support for socket tests. -The :mod:`test.support.socket_helper` module provides support for socket tests. +The :mod:`!test.support.socket_helper` module provides support for socket tests. .. versionadded:: 3.9 @@ -1161,14 +1165,14 @@ The :mod:`test.support.socket_helper` module provides support for socket tests. exceptions. -:mod:`test.support.script_helper` --- Utilities for the Python execution tests -============================================================================== +:mod:`!test.support.script_helper` --- Utilities for the Python execution tests +=============================================================================== .. module:: test.support.script_helper :synopsis: Support for Python's script execution tests. -The :mod:`test.support.script_helper` module provides support for Python's +The :mod:`!test.support.script_helper` module provides support for Python's script execution tests. .. function:: interpreter_requires_environment() @@ -1272,13 +1276,13 @@ script execution tests. path and the archive name for the zip file. -:mod:`test.support.bytecode_helper` --- Support tools for testing correct bytecode generation -============================================================================================= +:mod:`!test.support.bytecode_helper` --- Support tools for testing correct bytecode generation +============================================================================================== .. module:: test.support.bytecode_helper :synopsis: Support tools for testing correct bytecode generation. -The :mod:`test.support.bytecode_helper` module provides support for testing +The :mod:`!test.support.bytecode_helper` module provides support for testing and inspecting bytecode generation. .. versionadded:: 3.9 @@ -1304,13 +1308,13 @@ The module defines the following class: Throws :exc:`AssertionError` if *opname* is found. -:mod:`test.support.threading_helper` --- Utilities for threading tests -====================================================================== +:mod:`!test.support.threading_helper` --- Utilities for threading tests +======================================================================= .. module:: test.support.threading_helper :synopsis: Support for threading tests. -The :mod:`test.support.threading_helper` module provides support for threading tests. +The :mod:`!test.support.threading_helper` module provides support for threading tests. .. versionadded:: 3.10 @@ -1391,13 +1395,13 @@ The :mod:`test.support.threading_helper` module provides support for threading t finished. -:mod:`test.support.os_helper` --- Utilities for os tests -======================================================================== +:mod:`!test.support.os_helper` --- Utilities for os tests +========================================================= .. module:: test.support.os_helper :synopsis: Support for os tests. -The :mod:`test.support.os_helper` module provides support for os tests. +The :mod:`!test.support.os_helper` module provides support for os tests. .. versionadded:: 3.10 @@ -1586,13 +1590,13 @@ The :mod:`test.support.os_helper` module provides support for os tests. wrapped with a wait loop that checks for the existence of the file. -:mod:`test.support.import_helper` --- Utilities for import tests -================================================================ +:mod:`!test.support.import_helper` --- Utilities for import tests +================================================================= .. module:: test.support.import_helper :synopsis: Support for import tests. -The :mod:`test.support.import_helper` module provides support for import tests. +The :mod:`!test.support.import_helper` module provides support for import tests. .. versionadded:: 3.10 @@ -1700,13 +1704,13 @@ The :mod:`test.support.import_helper` module provides support for import tests. will be reverted at the end of the block. -:mod:`test.support.warnings_helper` --- Utilities for warnings tests -==================================================================== +:mod:`!test.support.warnings_helper` --- Utilities for warnings tests +===================================================================== .. module:: test.support.warnings_helper :synopsis: Support for warnings tests. -The :mod:`test.support.warnings_helper` module provides support for warnings tests. +The :mod:`!test.support.warnings_helper` module provides support for warnings tests. .. versionadded:: 3.10 diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index a58b460fef4..d12968dee91 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -4,14 +4,11 @@ .. module:: textwrap :synopsis: Text wrapping and filling -.. moduleauthor:: Greg Ward -.. sectionauthor:: Greg Ward - **Source code:** :source:`Lib/textwrap.py` -------------- -The :mod:`textwrap` module provides some convenience functions, +The :mod:`!textwrap` module provides some convenience functions, as well as :class:`TextWrapper`, the class that does all the work. If you're just wrapping or filling one or two text strings, the convenience functions should be good enough; otherwise, you should use an instance of @@ -102,6 +99,10 @@ functions should be good enough; otherwise, you should use an instance of print(repr(s)) # prints ' hello\n world\n ' print(repr(dedent(s))) # prints 'hello\n world\n' + .. versionchanged:: 3.14 + The :func:`!dedent` function now correctly normalizes blank lines containing + only whitespace characters. Previously, the implementation only normalized + blank lines containing tabs and spaces. .. function:: indent(text, prefix, predicate=None) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 9a0aeb7c128..fbe3951e034 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -198,7 +198,7 @@ This module defines the following functions: .. versionchanged:: 3.13 Added support for GNU/kFreeBSD. - .. versionchanged:: next + .. versionchanged:: 3.15 Added support for Solaris. @@ -608,7 +608,7 @@ since it is impossible to detect the termination of alien threads. timeout occurs. When the *timeout* argument is present and not ``None``, it should be a - floating-point number specifying a timeout for the operation in seconds + real number specifying a timeout for the operation in seconds (or fractions thereof). As :meth:`~Thread.join` always returns ``None``, you must call :meth:`~Thread.is_alive` after :meth:`~Thread.join` to decide whether a timeout happened -- if the thread is still alive, the @@ -632,6 +632,9 @@ since it is impossible to detect the termination of alien threads. May raise :exc:`PythonFinalizationError`. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. attribute:: name A string used for identification purposes only. It has no semantics. @@ -764,7 +767,7 @@ All methods are executed atomically. If a call with *blocking* set to ``True`` would block, return ``False`` immediately; otherwise, set the lock to locked and return ``True``. - When invoked with the floating-point *timeout* argument set to a positive + When invoked with the *timeout* argument set to a positive value, block for at most the number of seconds specified by *timeout* and as long as the lock cannot be acquired. A *timeout* argument of ``-1`` specifies an unbounded wait. It is forbidden to specify a *timeout* @@ -783,6 +786,9 @@ All methods are executed atomically. .. versionchanged:: 3.14 Lock acquisition can now be interrupted by signals on Windows. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. method:: release() @@ -863,7 +869,7 @@ call release as many times the lock has been acquired can lead to deadlock. * If no thread owns the lock, acquire the lock and return immediately. * If another thread owns the lock, block until we are able to acquire - lock, or *timeout*, if set to a positive float value. + lock, or *timeout*, if set to a positive value. * If the same thread owns the lock, acquire the lock again, and return immediately. This is the difference between :class:`Lock` and @@ -890,6 +896,9 @@ call release as many times the lock has been acquired can lead to deadlock. .. versionchanged:: 3.2 The *timeout* parameter is new. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. method:: release() @@ -1023,7 +1032,7 @@ item to the buffer only needs to wake up one consumer thread. occurs. Once awakened or timed out, it re-acquires the lock and returns. When the *timeout* argument is present and not ``None``, it should be a - floating-point number specifying a timeout for the operation in seconds + real number specifying a timeout for the operation in seconds (or fractions thereof). When the underlying lock is an :class:`RLock`, it is not released using @@ -1150,6 +1159,9 @@ Semaphores also support the :ref:`context management protocol `. .. versionchanged:: 3.2 The *timeout* parameter is new. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. method:: release(n=1) Release a semaphore, incrementing the internal counter by *n*. When it @@ -1250,7 +1262,7 @@ method. The :meth:`~Event.wait` method blocks until the flag is true. the internal flag did not become true within the given wait time. When the timeout argument is present and not ``None``, it should be a - floating-point number specifying a timeout for the operation in seconds, + real number specifying a timeout for the operation in seconds, or fractions thereof. .. versionchanged:: 3.1 @@ -1424,3 +1436,159 @@ is equivalent to:: Currently, :class:`Lock`, :class:`RLock`, :class:`Condition`, :class:`Semaphore`, and :class:`BoundedSemaphore` objects may be used as :keyword:`with` statement context managers. + + +Iterator synchronization +------------------------ + +By default, Python iterators do not support concurrent access. Most iterators make +no guarantees when accessed simultaneously from multiple threads. Generator +iterators, for example, raise :exc:`ValueError` if one of their iterator methods +is called while the generator is already executing. The tools in this section +allow reliable concurrency support to be added to ordinary iterators and +iterator-producing callables. + +The :class:`serialize_iterator` wrapper lets multiple threads share a single iterator and +take turns consuming from it. While one thread is running ``__next__()``, the +others block until the iterator becomes available. Each value produced by the +underlying iterator is delivered to exactly one caller. + +The :func:`concurrent_tee` function lets multiple threads each receive the full +stream of values from one underlying iterator. It creates independent iterators +that all draw from the same source. Values are buffered until consumed by all +of the derived iterators. + +.. class:: serialize_iterator(iterable) + + Return an iterator wrapper that serializes concurrent calls to + :meth:`~iterator.__next__` using a lock. + + If the wrapped iterator also defines :meth:`~generator.send`, + :meth:`~generator.throw`, or :meth:`~generator.close`, those calls + are serialized as well. + + This makes it possible to share a single iterator, including a generator + iterator, between multiple threads. A lock ensures that calls are handled + one at a time. No values are duplicated or skipped by the wrapper itself. + Each item from the underlying iterator is given to exactly one caller. + + This wrapper does not copy or buffer values. Threads that call + :func:`next` while another thread is already advancing the iterator will + block until the active call completes. + + Example: + + .. code-block:: python + + import threading + + def squares(n): + for x in range(n): + yield x * x + + def consume(name, iterable): + for item in iterable: + print(name, item) + + source = threading.serialize_iterator(squares(5)) + + t1 = threading.Thread(target=consume, args=("left", source)) + t2 = threading.Thread(target=consume, args=("right", source)) + t1.start() + t2.start() + t1.join() + t2.join() + + In this example, each number is printed exactly once, but the work is shared + between the two threads. + + .. versionadded:: next + + +.. function:: synchronized_iterator(func) + + Wrap an iterator-producing callable so that each iterator it returns is + automatically passed through :class:`serialize_iterator`. + + This is especially useful as a :term:`decorator` for generator functions, + allowing their generator-iterators to be consumed from multiple threads. + + Example: + + .. code-block:: python + + import threading + + @threading.synchronized_iterator + def squares(n): + for x in range(n): + yield x * x + + def consume(name, iterable): + for item in iterable: + print(name, item) + + source = squares(5) + + t1 = threading.Thread(target=consume, args=("left", source)) + t2 = threading.Thread(target=consume, args=("right", source)) + t1.start() + t2.start() + t1.join() + t2.join() + + The returned wrapper preserves the metadata of *func*, such as its name and + wrapped function reference. + + .. versionadded:: next + + +.. function:: concurrent_tee(iterable, n=2) + + Return *n* independent iterators from a single input *iterable*, with + guaranteed behavior when the derived iterators are consumed concurrently. + + This function is similar to :func:`itertools.tee`, but is intended for cases + where the source iterator may feed consumers running in different threads. + Each returned iterator yields every value from the underlying iterable, in + the same order. + + Internally, values are buffered until every derived iterator has consumed + them. + + The returned iterators share the same underlying synchronization lock. Each + individual derived iterator is intended to be consumed by one thread at a + time. If a single derived iterator must itself be shared by multiple + threads, wrap it with :class:`serialize_iterator`. + + If *n* is ``0``, return an empty tuple. If *n* is negative, raise + :exc:`ValueError`. + + Example: + + .. code-block:: python + + import threading + + def squares(n): + for x in range(n): + yield x * x + + def consume(name, iterable): + for item in iterable: + print(name, item) + + source = squares(5) + left, right = threading.concurrent_tee(source) + + t1 = threading.Thread(target=consume, args=("left", left)) + t2 = threading.Thread(target=consume, args=("right", right)) + t1.start() + t2.start() + t1.join() + t2.join() + + In this example, both consumer threads see the full sequence of squares + from a single generator expression. + + .. versionadded:: next diff --git a/Doc/library/threadsafety.rst b/Doc/library/threadsafety.rst new file mode 100644 index 00000000000..a529f7803af --- /dev/null +++ b/Doc/library/threadsafety.rst @@ -0,0 +1,606 @@ +.. _threadsafety: + +************************ +Thread Safety Guarantees +************************ + +This page documents thread-safety guarantees for built-in types in Python's +free-threaded build. The guarantees described here apply when using Python with +the :term:`GIL` disabled (free-threaded mode). When the GIL is enabled, most +operations are implicitly serialized. + +For general guidance on writing thread-safe code in free-threaded Python, see +:ref:`freethreading-python-howto`. + + +.. _threadsafety-levels: + +Thread safety levels +==================== + +The C API documentation uses the following levels to describe the thread +safety guarantees of each function. The levels are listed from least to +most safe. + +.. _threadsafety-level-incompatible: + +Incompatible +------------ + +A function or operation that cannot be made safe for concurrent use even +with external synchronization. Incompatible code typically accesses +global state in an unsynchronized way and must only be called from a single +thread throughout the program's lifetime. + +Example: a function that modifies process-wide state such as signal handlers +or environment variables, where concurrent calls from any threads, even with +external locking, can conflict with the runtime or other libraries. + +.. _threadsafety-level-compatible: + +Compatible +---------- + +A function or operation that is safe to call from multiple threads +*provided* the caller supplies appropriate external synchronization, for +example by holding a :term:`lock` for the duration of each call. Without +such synchronization, concurrent calls may produce :term:`race conditions +` or :term:`data races `. + +Example: a function that reads from or writes to an object whose internal +state is not protected by a lock. Callers must ensure that no two threads +access the same object at the same time. + +.. _threadsafety-level-distinct: + +Safe on distinct objects +------------------------ + +A function or operation that is safe to call from multiple threads without +external synchronization, as long as each thread operates on a **different** +object. Two threads may call the function at the same time, but they must +not pass the same object (or objects that share underlying state) as +arguments. + +Example: a function that modifies fields of a struct using non-atomic +writes. Two threads can each call the function on their own struct +instance safely, but concurrent calls on the *same* instance require +external synchronization. + +.. _threadsafety-level-shared: + +Safe on shared objects +---------------------- + +A function or operation that is safe for concurrent use on the **same** +object. The implementation uses internal synchronization (such as +:term:`per-object locks ` or +:ref:`critical sections `) to protect shared +mutable state, so callers do not need to supply their own locking. + +Example: :c:func:`PyList_GetItemRef` can be called from multiple threads on the +same :c:type:`PyListObject` - it uses internal synchronization to serialize +access. + +.. _threadsafety-level-atomic: + +Atomic +------ + +A function or operation that appears :term:`atomic ` with +respect to other threads - it executes instantaneously from the perspective +of other threads. This is the strongest form of thread safety. + +Example: :c:func:`PyMutex_IsLocked` performs an atomic read of the mutex +state and can be called from any thread at any time. + + +.. _thread-safety-list: + +Thread safety for list objects +============================== + +Reading a single element from a :class:`list` is +:term:`atomic `: + +.. code-block:: + :class: good + + lst[i] # list.__getitem__ + +The following methods traverse the list and use :term:`atomic ` +reads of each item to perform their function. That means that they may +return results affected by concurrent modifications: + +.. code-block:: + :class: maybe + + item in lst + lst.index(item) + lst.count(item) + +All of the above operations avoid acquiring :term:`per-object locks +`. They do not block concurrent modifications. Other +operations that hold a lock will not block these from observing intermediate +states. + +All other operations from here on block using the :term:`per-object lock`. + +Writing a single item via ``lst[i] = x`` is safe to call from multiple +threads and will not corrupt the list. + +The following operations return new objects and appear +:term:`atomic ` to other threads: + +.. code-block:: + :class: good + + lst1 + lst2 # concatenates two lists into a new list + x * lst # repeats lst x times into a new list + lst.copy() # returns a shallow copy of the list + +The following methods that only operate on a single element with no shifting +required are :term:`atomic `: + +.. code-block:: + :class: good + + lst.append(x) # append to the end of the list, no shifting required + lst.pop() # pop element from the end of the list, no shifting required + +The :meth:`~list.clear` method is also :term:`atomic `. +Other threads cannot observe elements being removed. + +The :meth:`~list.sort` method is not :term:`atomic `. +Other threads cannot observe intermediate states during sorting, but the +list appears empty for the duration of the sort. + +The following operations may allow :term:`lock-free` operations to observe +intermediate states since they modify multiple elements in place: + +.. code-block:: + :class: maybe + + lst.insert(idx, item) # shifts elements + lst.pop(idx) # idx not at the end of the list, shifts elements + lst *= x # copies elements in place + +The :meth:`~list.remove` method may allow concurrent modifications since +element comparison may execute arbitrary Python code (via +:meth:`~object.__eq__`). + +:meth:`~list.extend` is safe to call from multiple threads. However, its +guarantees depend on the iterable passed to it. If it is a :class:`list`, a +:class:`tuple`, a :class:`set`, a :class:`frozenset`, a :class:`dict` or a +:ref:`dictionary view object ` (but not their subclasses), the +``extend`` operation is safe from concurrent modifications to the iterable. +Otherwise, an iterator is created which can be concurrently modified by +another thread. The same applies to inplace concatenation of a list with +other iterables when using ``lst += iterable``. + +Similarly, assigning to a list slice with ``lst[i:j] = iterable`` is safe +to call from multiple threads, but ``iterable`` is only locked when it is +also a :class:`list` (but not its subclasses). + +Operations that involve multiple accesses, as well as iteration, are never +atomic. For example: + +.. code-block:: + :class: bad + + # NOT atomic: read-modify-write + lst[i] = lst[i] + 1 + + # NOT atomic: check-then-act + if lst: + item = lst.pop() + + # NOT thread-safe: iteration while modifying + for item in lst: + process(item) # another thread may modify lst + +Consider external synchronization when sharing :class:`list` instances +across threads. + + +.. _thread-safety-dict: + +Thread safety for dict objects +============================== + +Creating a dictionary with the :class:`dict` constructor is atomic when the +argument to it is a :class:`dict` or a :class:`tuple`. When using the +:meth:`dict.fromkeys` method, dictionary creation is atomic when the +argument is a :class:`dict`, :class:`tuple`, :class:`set` or +:class:`frozenset`. + +The following operations and functions are :term:`lock-free` and +:term:`atomic `. + +.. code-block:: + :class: good + + d[key] # dict.__getitem__ + d.get(key) # dict.get + key in d # dict.__contains__ + len(d) # dict.__len__ + +All other operations from here on hold the :term:`per-object lock`. + +Writing or removing a single item is safe to call from multiple threads +and will not corrupt the dictionary: + +.. code-block:: + :class: good + + d[key] = value # write + del d[key] # delete + d.pop(key) # remove and return + d.popitem() # remove and return last item + d.setdefault(key, v) # insert if missing + +These operations may compare keys using :meth:`~object.__eq__`, which can +execute arbitrary Python code. During such comparisons, the dictionary may +be modified by another thread. For built-in types like :class:`str`, +:class:`int`, and :class:`float`, that implement :meth:`~object.__eq__` in C, +the underlying lock is not released during comparisons and this is not a +concern. + +The following operations return new objects and hold the :term:`per-object lock` +for the duration of the operation: + +.. code-block:: + :class: good + + d.copy() # returns a shallow copy of the dictionary + d | other # merges two dicts into a new dict + d.keys() # returns a new dict_keys view object + d.values() # returns a new dict_values view object + d.items() # returns a new dict_items view object + +The :meth:`~dict.clear` method holds the lock for its duration. Other +threads cannot observe elements being removed. + +The following operations lock both dictionaries. For :meth:`~dict.update` +and ``|=``, this applies only when the other operand is a :class:`dict` +that uses the standard dict iterator (but not subclasses that override +iteration). For equality comparison, this applies to :class:`dict` and +its subclasses: + +.. code-block:: + :class: good + + d.update(other_dict) # both locked when other_dict is a dict + d |= other_dict # both locked when other_dict is a dict + d == other_dict # both locked for dict and subclasses + +All comparison operations also compare values using :meth:`~object.__eq__`, +so for non-built-in types the lock may be released during comparison. + +:meth:`~dict.fromkeys` locks both the new dictionary and the iterable +when the iterable is exactly a :class:`dict`, :class:`set`, or +:class:`frozenset` (not subclasses): + +.. code-block:: + :class: good + + dict.fromkeys(a_dict) # locks both + dict.fromkeys(a_set) # locks both + dict.fromkeys(a_frozenset) # locks both + +When updating from a non-dict iterable, only the target dictionary is +locked. The iterable may be concurrently modified by another thread: + +.. code-block:: + :class: maybe + + d.update(iterable) # iterable is not a dict: only d locked + d |= iterable # iterable is not a dict: only d locked + dict.fromkeys(iterable) # iterable is not a dict/set/frozenset: only result locked + +Operations that involve multiple accesses, as well as iteration, are never +atomic: + +.. code-block:: + :class: bad + + # NOT atomic: read-modify-write + d[key] = d[key] + 1 + + # NOT atomic: check-then-act (TOCTOU) + if key in d: + del d[key] + + # NOT thread-safe: iteration while modifying + for key, value in d.items(): + process(key) # another thread may modify d + +To avoid time-of-check to time-of-use (TOCTOU) issues, use atomic +operations or handle exceptions: + +.. code-block:: + :class: good + + # Use pop() with default instead of check-then-delete + d.pop(key, None) + + # Or handle the exception + try: + del d[key] + except KeyError: + pass + +To safely iterate over a dictionary that may be modified by another +thread, iterate over a copy: + +.. code-block:: + :class: good + + # Make a copy to iterate safely + for key, value in d.copy().items(): + process(key) + +Consider external synchronization when sharing :class:`dict` instances +across threads. + + +.. _thread-safety-set: + +Thread safety for set objects +============================== + +The :func:`len` function is lock-free and :term:`atomic `. + +The following read operation is lock-free. It does not block concurrent +modifications and may observe intermediate states from operations that +hold the per-object lock: + +.. code-block:: + :class: good + + elem in s # set.__contains__ + +This operation may compare elements using :meth:`~object.__eq__`, which can +execute arbitrary Python code. During such comparisons, the set may be +modified by another thread. For built-in types like :class:`str`, +:class:`int`, and :class:`float`, :meth:`!__eq__` does not release the +underlying lock during comparisons and this is not a concern. + +All other operations from here on hold the per-object lock. + +Adding or removing a single element is safe to call from multiple threads +and will not corrupt the set: + +.. code-block:: + :class: good + + s.add(elem) # add element + s.remove(elem) # remove element, raise if missing + s.discard(elem) # remove element if present + s.pop() # remove and return arbitrary element + +These operations also compare elements, so the same :meth:`~object.__eq__` +considerations as above apply. + +The :meth:`~set.copy` method returns a new object and holds the per-object lock +for the duration so that it is always atomic. + +The :meth:`~set.clear` method holds the lock for its duration. Other +threads cannot observe elements being removed. + +The following operations only accept :class:`set` or :class:`frozenset` +as operands and always lock both objects: + +.. code-block:: + :class: good + + s |= other # other must be set/frozenset + s &= other # other must be set/frozenset + s -= other # other must be set/frozenset + s ^= other # other must be set/frozenset + s & other # other must be set/frozenset + s | other # other must be set/frozenset + s - other # other must be set/frozenset + s ^ other # other must be set/frozenset + +:meth:`set.update`, :meth:`set.union`, :meth:`set.intersection` and +:meth:`set.difference` can take multiple iterables as arguments. They all +iterate through all the passed iterables and do the following: + + * :meth:`set.update` and :meth:`set.union` lock both objects only when + the other operand is a :class:`set`, :class:`frozenset`, or :class:`dict`. + * :meth:`set.intersection` and :meth:`set.difference` always try to lock + all objects. + +:meth:`set.symmetric_difference` tries to lock both objects. + +The update variants of the above methods also have some differences between +them: + + * :meth:`set.difference_update` and :meth:`set.intersection_update` try + to lock all objects one-by-one. + * :meth:`set.symmetric_difference_update` only locks the arguments if it is + of type :class:`set`, :class:`frozenset`, or :class:`dict`. + +The following methods always try to lock both objects: + +.. code-block:: + :class: good + + s.isdisjoint(other) # both locked + s.issubset(other) # both locked + s.issuperset(other) # both locked + +Operations that involve multiple accesses, as well as iteration, are never +atomic: + +.. code-block:: + :class: bad + + # NOT atomic: check-then-act + if elem in s: + s.remove(elem) + + # NOT thread-safe: iteration while modifying + for elem in s: + process(elem) # another thread may modify s + +Consider external synchronization when sharing :class:`set` instances +across threads. See :ref:`freethreading-python-howto` for more information. + + +.. _thread-safety-bytearray: + +Thread safety for bytearray objects +=================================== + + The :func:`len` function is lock-free and :term:`atomic `. + + Concatenation and comparisons use the buffer protocol, which prevents + resizing but does not hold the per-object lock. These operations may + observe intermediate states from concurrent modifications: + + .. code-block:: + :class: maybe + + ba + other # may observe concurrent writes + ba == other # may observe concurrent writes + ba < other # may observe concurrent writes + + All other operations from here on hold the per-object lock. + + Reading a single element or slice is safe to call from multiple threads: + + .. code-block:: + :class: good + + ba[i] # bytearray.__getitem__ + ba[i:j] # slice + + The following operations are safe to call from multiple threads and will + not corrupt the bytearray: + + .. code-block:: + :class: good + + ba[i] = x # write single byte + ba[i:j] = values # write slice + ba.append(x) # append single byte + ba.extend(other) # extend with iterable + ba.insert(i, x) # insert single byte + ba.pop() # remove and return last byte + ba.pop(i) # remove and return byte at index + ba.remove(x) # remove first occurrence + ba.reverse() # reverse in place + ba.clear() # remove all bytes + + Slice assignment locks both objects when *values* is a :class:`bytearray`: + + .. code-block:: + :class: good + + ba[i:j] = other_bytearray # both locked + + The following operations return new objects and hold the per-object lock + for the duration: + + .. code-block:: + :class: good + + ba.copy() # returns a shallow copy + ba * n # repeat into new bytearray + + The membership test holds the lock for its duration: + + .. code-block:: + :class: good + + x in ba # bytearray.__contains__ + + All other bytearray methods (such as :meth:`~bytearray.find`, + :meth:`~bytearray.replace`, :meth:`~bytearray.split`, + :meth:`~bytearray.decode`, etc.) hold the per-object lock for their + duration. + + Operations that involve multiple accesses, as well as iteration, are never + atomic: + + .. code-block:: + :class: bad + + # NOT atomic: check-then-act + if x in ba: + ba.remove(x) + + # NOT thread-safe: iteration while modifying + for byte in ba: + process(byte) # another thread may modify ba + + To safely iterate over a bytearray that may be modified by another + thread, iterate over a copy: + + .. code-block:: + :class: good + + # Make a copy to iterate safely + for byte in ba.copy(): + process(byte) + + Consider external synchronization when sharing :class:`bytearray` instances + across threads. See :ref:`freethreading-python-howto` for more information. + + +.. _thread-safety-memoryview: + +Thread safety for memoryview objects +==================================== + +:class:`memoryview` objects provide access to the internal data of an +underlying object without copying. Thread safety depends on both the +memoryview itself and the underlying buffer exporter. + +The memoryview implementation uses atomic operations to track its own +exports in the :term:`free-threaded build`. Creating and +releasing a memoryview are thread-safe. Attribute access (e.g., +:attr:`~memoryview.shape`, :attr:`~memoryview.format`) reads fields that +are immutable for the lifetime of the memoryview, so concurrent reads +are safe as long as the memoryview has not been released. + +However, the actual data accessed through the memoryview is owned by the +underlying object. Concurrent access to this data is only safe if the +underlying object supports it: + +* For immutable objects like :class:`bytes`, concurrent reads through + multiple memoryviews are safe. + +* For mutable objects like :class:`bytearray`, reading and writing the + same memory region from multiple threads without external + synchronization is not safe and may result in data corruption. + Note that even read-only memoryviews of mutable objects do not + prevent data races if the underlying object is modified from + another thread. + +.. code-block:: + :class: bad + + # NOT safe: concurrent writes to the same buffer + data = bytearray(1000) + view = memoryview(data) + # Thread 1: view[0:500] = b'x' * 500 + # Thread 2: view[0:500] = b'y' * 500 + +.. code-block:: + :class: good + + # Safe: use a lock for concurrent access + import threading + lock = threading.Lock() + data = bytearray(1000) + view = memoryview(data) + + with lock: + view[0:500] = b'x' * 500 + +Resizing or reallocating the underlying object (such as calling +:meth:`bytearray.resize`) while a memoryview is exported raises +:exc:`BufferError`. This is enforced regardless of threading. diff --git a/Doc/library/time.rst b/Doc/library/time.rst index b05c0a312db..a931134331f 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -189,7 +189,7 @@ Functions .. versionadded:: 3.7 -.. function:: clock_settime(clk_id, time: float) +.. function:: clock_settime(clk_id, time) Set the time of the specified clock *clk_id*. Currently, :data:`CLOCK_REALTIME` is the only accepted value for *clk_id*. @@ -201,6 +201,9 @@ Functions .. versionadded:: 3.3 + .. versionchanged:: 3.15 + Accepts any real number as *time*, not only integer or float. + .. function:: clock_settime_ns(clk_id, time: int) @@ -223,6 +226,9 @@ Functions ``asctime(localtime(secs))``. Locale information is not used by :func:`ctime`. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. + .. function:: get_clock_info(name) @@ -238,8 +244,8 @@ Functions The result has the following attributes: - - *adjustable*: ``True`` if the clock can be changed automatically (e.g. by - a NTP daemon) or manually by the system administrator, ``False`` otherwise + - *adjustable*: ``True`` if the clock can be set to jump forward or backward + in time, ``False`` otherwise. Does not refer to gradual NTP rate adjustments. - *implementation*: The name of the underlying C function used to get the clock value. Refer to :ref:`time-clock-id-constants` for possible values. - *monotonic*: ``True`` if the clock cannot go backward, @@ -258,6 +264,9 @@ Functions :class:`struct_time` object. See :func:`calendar.timegm` for the inverse of this function. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. + .. function:: localtime([secs]) @@ -271,6 +280,9 @@ Functions :c:func:`gmtime` failure. It's common for this to be restricted to years between 1970 and 2038. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. + .. function:: mktime(t) @@ -382,8 +394,7 @@ Functions .. function:: sleep(secs) Suspend execution of the calling thread for the given number of seconds. - The argument may be a floating-point number to indicate a more precise sleep - time. + The argument may be a non-integer to indicate a more precise sleep time. If the sleep is interrupted by a signal and no exception is raised by the signal handler, the sleep is restarted with a recomputed timeout. @@ -396,9 +407,9 @@ Functions On Windows, if *secs* is zero, the thread relinquishes the remainder of its time slice to any other thread that is ready to run. If there are no other threads ready to run, the function returns immediately, and the thread - continues execution. On Windows 8.1 and newer the implementation uses + continues execution. On Windows 10 and newer the implementation uses a `high-resolution timer - `_ + `_ which provides resolution of 100 nanoseconds. If *secs* is zero, ``Sleep(0)`` is used. .. rubric:: Unix implementation @@ -428,6 +439,9 @@ Functions .. versionchanged:: 3.13 Raises an auditing event. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. + .. index:: single: % (percent); datetime format @@ -570,7 +584,7 @@ Functions calculations when the day of the week and the year are specified. Here is an example, a format for dates compatible with that specified in the - :rfc:`2822` Internet email standard. [1]_ :: + :rfc:`5322` Internet email standard. [1]_ :: >>> from time import gmtime, strftime >>> strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()) @@ -1052,4 +1066,5 @@ Timezone Constants strict reading of the original 1982 :rfc:`822` standard calls for a two-digit year (``%y`` rather than ``%Y``), but practice moved to 4-digit years long before the year 2000. After that, :rfc:`822` became obsolete and the 4-digit year has - been first recommended by :rfc:`1123` and then mandated by :rfc:`2822`. + been first recommended by :rfc:`1123` and then mandated by :rfc:`2822`, + with :rfc:`5322` continuing this requirement. diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 548a3ee0540..fd67c5c0a0f 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -143,21 +143,24 @@ The module defines three convenience functions and a public class: timeit.Timer('for i in range(10): oct(i)', 'gc.enable()').timeit() - .. method:: Timer.autorange(callback=None) + .. method:: Timer.autorange(callback=None, target_time=None) Automatically determine how many times to call :meth:`.timeit`. This is a convenience function that calls :meth:`.timeit` repeatedly - so that the total time >= 0.2 second, returning the eventual + so that the total time >= *Timer.target_time* seconds, returning the eventual (number of loops, time taken for that number of loops). It calls :meth:`.timeit` with increasing numbers from the sequence 1, 2, 5, - 10, 20, 50, ... until the time taken is at least 0.2 seconds. + 10, 20, 50, ... until the time taken is at least *target_time* seconds. If *callback* is given and is not ``None``, it will be called after each trial with two arguments: ``callback(number, time_taken)``. .. versionadded:: 3.6 + .. versionchanged:: 3.15 + The optional *target_time* parameter was added. + .. method:: Timer.repeat(repeat=5, number=1000000) @@ -239,6 +242,13 @@ Where the following options are understood: .. versionadded:: 3.5 +.. option:: -t, --target-time=T + + if :option:`--number` is 0, the code will run until it takes at + least this many seconds (default: 0.2) + + .. versionadded:: 3.15 + .. option:: -v, --verbose print raw timing results; repeat for more digits precision @@ -254,7 +264,7 @@ similarly. If :option:`-n` is not given, a suitable number of loops is calculated by trying increasing numbers from the sequence 1, 2, 5, 10, 20, 50, ... until the total -time is at least 0.2 seconds. +time is at least :option:`--target-time` seconds (default: 0.2). :func:`default_timer` measurements can be affected by other programs running on the same machine, so the best thing to do when accurate timing is necessary is @@ -355,7 +365,7 @@ to test for missing and present object attributes: 0.08588060699912603 -To give the :mod:`timeit` module access to functions you define, you can pass a +To give the :mod:`!timeit` module access to functions you define, you can pass a *setup* parameter which contains an import statement:: def test(): diff --git a/Doc/library/tkinter.colorchooser.rst b/Doc/library/tkinter.colorchooser.rst index df2b324fd5d..73f8f76a210 100644 --- a/Doc/library/tkinter.colorchooser.rst +++ b/Doc/library/tkinter.colorchooser.rst @@ -2,14 +2,13 @@ ====================================================== .. module:: tkinter.colorchooser - :platform: Tk :synopsis: Color choosing dialog **Source code:** :source:`Lib/tkinter/colorchooser.py` -------------- -The :mod:`tkinter.colorchooser` module provides the :class:`Chooser` class +The :mod:`!tkinter.colorchooser` module provides the :class:`Chooser` class as an interface to the native color picker dialog. ``Chooser`` implements a modal color choosing dialog window. The ``Chooser`` class inherits from the :class:`~tkinter.commondialog.Dialog` class. diff --git a/Doc/library/tkinter.dnd.rst b/Doc/library/tkinter.dnd.rst index 62298d96c26..48d16ccb204 100644 --- a/Doc/library/tkinter.dnd.rst +++ b/Doc/library/tkinter.dnd.rst @@ -2,7 +2,6 @@ ============================================= .. module:: tkinter.dnd - :platform: Tk :synopsis: Tkinter drag-and-drop interface **Source code:** :source:`Lib/tkinter/dnd.py` @@ -12,7 +11,7 @@ .. note:: This is experimental and due to be deprecated when it is replaced with the Tk DND. -The :mod:`tkinter.dnd` module provides drag-and-drop support for objects within +The :mod:`!tkinter.dnd` module provides drag-and-drop support for objects within a single application, within the same window or between windows. To enable an object to be dragged, you must create an event binding for it that starts the drag-and-drop process. Typically, you bind a ButtonPress event to a callback diff --git a/Doc/library/tkinter.font.rst b/Doc/library/tkinter.font.rst index ed01bd5f483..9eecb803c3a 100644 --- a/Doc/library/tkinter.font.rst +++ b/Doc/library/tkinter.font.rst @@ -2,14 +2,13 @@ ============================================= .. module:: tkinter.font - :platform: Tk :synopsis: Tkinter font-wrapping class **Source code:** :source:`Lib/tkinter/font.py` -------------- -The :mod:`tkinter.font` module provides the :class:`Font` class for creating +The :mod:`!tkinter.font` module provides the :class:`Font` class for creating and using named fonts. The different font weights and slants are: diff --git a/Doc/library/tkinter.messagebox.rst b/Doc/library/tkinter.messagebox.rst index 0dc9632ca73..2a69d282638 100644 --- a/Doc/library/tkinter.messagebox.rst +++ b/Doc/library/tkinter.messagebox.rst @@ -2,14 +2,13 @@ ====================================================== .. module:: tkinter.messagebox - :platform: Tk :synopsis: Various types of alert dialogs **Source code:** :source:`Lib/tkinter/messagebox.py` -------------- -The :mod:`tkinter.messagebox` module provides a template base class as well as +The :mod:`!tkinter.messagebox` module provides a template base class as well as a variety of convenience methods for commonly used configurations. The message boxes are modal and will return a subset of (``True``, ``False``, ``None``, :data:`OK`, :data:`CANCEL`, :data:`YES`, :data:`NO`) based on diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index f284988daf2..a34b74a0888 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -4,18 +4,16 @@ .. module:: tkinter :synopsis: Interface to Tcl/Tk for graphical user interfaces -.. moduleauthor:: Guido van Rossum - **Source code:** :source:`Lib/tkinter/__init__.py` -------------- -The :mod:`tkinter` package ("Tk interface") is the standard Python interface to -the Tcl/Tk GUI toolkit. Both Tk and :mod:`tkinter` are available on most Unix +The :mod:`!tkinter` package ("Tk interface") is the standard Python interface to +the Tcl/Tk GUI toolkit. Both Tk and :mod:`!tkinter` are available on most Unix platforms, including macOS, as well as on Windows systems. Running ``python -m tkinter`` from the command line should open a window -demonstrating a simple Tk interface, letting you know that :mod:`tkinter` is +demonstrating a simple Tk interface, letting you know that :mod:`!tkinter` is properly installed on your system, and also showing what version of Tcl/Tk is installed, so you can read the Tcl/Tk documentation specific to that version. @@ -36,6 +34,8 @@ details that are unchanged. Most documentation you will find online still uses the old API and can be woefully outdated. +.. include:: ../includes/optional-module.rst + .. seealso:: * `TkDocs `_ @@ -106,7 +106,7 @@ Internally, Tk and Ttk use facilities of the underlying operating system, i.e., Xlib on Unix/X11, Cocoa on macOS, GDI on Windows. When your Python application uses a class in Tkinter, e.g., to create a widget, -the :mod:`tkinter` module first assembles a Tcl/Tk command string. It passes that +the :mod:`!tkinter` module first assembles a Tcl/Tk command string. It passes that Tcl command string to an internal :mod:`_tkinter` binary module, which then calls the Tcl interpreter to evaluate it. The Tcl interpreter will then call into the Tk and/or Ttk packages, which will in turn make calls to Xlib, Cocoa, or GDI. @@ -116,7 +116,7 @@ Tkinter Modules --------------- Support for Tkinter is spread across several modules. Most applications will need the -main :mod:`tkinter` module, as well as the :mod:`tkinter.ttk` module, which provides +main :mod:`!tkinter` module, as well as the :mod:`tkinter.ttk` module, which provides the modern themed widget set and API:: @@ -175,12 +175,12 @@ the modern themed widget set and API:: .. attribute:: master The widget object that contains this widget. For :class:`Tk`, the - *master* is :const:`None` because it is the main window. The terms + :attr:`!master` is :const:`None` because it is the main window. The terms *master* and *parent* are similar and sometimes used interchangeably as argument names; however, calling :meth:`winfo_parent` returns a - string of the widget name whereas :attr:`master` returns the object. + string of the widget name whereas :attr:`!master` returns the object. *parent*/*child* reflects the tree-like relationship while - *master*/*slave* reflects the container structure. + *master* (or *container*)/*content* reflects the container structure. .. attribute:: children @@ -202,7 +202,7 @@ the modern themed widget set and API:: The modules that provide Tk support include: -:mod:`tkinter` +:mod:`!tkinter` Main Tkinter module. :mod:`tkinter.colorchooser` @@ -228,7 +228,7 @@ The modules that provide Tk support include: :mod:`tkinter.ttk` Themed widget set introduced in Tk 8.5, providing modern alternatives - for many of the classic widgets in the main :mod:`tkinter` module. + for many of the classic widgets in the main :mod:`!tkinter` module. Additional modules: @@ -237,22 +237,22 @@ Additional modules: :mod:`_tkinter` A binary module that contains the low-level interface to Tcl/Tk. - It is automatically imported by the main :mod:`tkinter` module, + It is automatically imported by the main :mod:`!tkinter` module, and should never be used directly by application programmers. It is usually a shared library (or DLL), but might in some cases be statically linked with the Python interpreter. :mod:`idlelib` Python's Integrated Development and Learning Environment (IDLE). Based - on :mod:`tkinter`. + on :mod:`!tkinter`. :mod:`tkinter.constants` Symbolic constants that can be used in place of strings when passing various parameters to Tkinter calls. Automatically imported by the - main :mod:`tkinter` module. + main :mod:`!tkinter` module. :mod:`tkinter.dnd` - (experimental) Drag-and-drop support for :mod:`tkinter`. This will + (experimental) Drag-and-drop support for :mod:`!tkinter`. This will become deprecated when it is replaced with the Tk DND. :mod:`turtle` @@ -392,7 +392,7 @@ by spaces. Without getting into too many details, notice the following: * Operations which are implemented as separate *commands* in Tcl (like ``grid`` or ``destroy``) are represented as *methods* on Tkinter widget objects. As you'll see shortly, at other times Tcl uses what appear to be - method calls on widget objects, which more closely mirror what would is + method calls on widget objects, which more closely mirror what is used in Tkinter. @@ -502,7 +502,7 @@ documentation for all of these in the Threading model --------------- -Python and Tcl/Tk have very different threading models, which :mod:`tkinter` +Python and Tcl/Tk have very different threading models, which :mod:`!tkinter` tries to bridge. If you use threads, you may need to be aware of this. A Python interpreter may have many threads associated with it. In Tcl, multiple @@ -510,9 +510,9 @@ threads can be created, but each thread has a separate Tcl interpreter instance associated with it. Threads can also create more than one interpreter instance, though each interpreter instance can be used only by the one thread that created it. -Each :class:`Tk` object created by :mod:`tkinter` contains a Tcl interpreter. +Each :class:`Tk` object created by :mod:`!tkinter` contains a Tcl interpreter. It also keeps track of which thread created that interpreter. Calls to -:mod:`tkinter` can be made from any Python thread. Internally, if a call comes +:mod:`!tkinter` can be made from any Python thread. Internally, if a call comes from a thread other than the one that created the :class:`Tk` object, an event is posted to the interpreter's event queue, and when executed, the result is returned to the calling Python thread. @@ -527,17 +527,17 @@ toolkits where the GUI runs in a completely separate thread from all application code including event handlers. If the Tcl interpreter is not running the event loop and processing events, any -:mod:`tkinter` calls made from threads other than the one running the Tcl +:mod:`!tkinter` calls made from threads other than the one running the Tcl interpreter will fail. A number of special cases exist: * Tcl/Tk libraries can be built so they are not thread-aware. In this case, - :mod:`tkinter` calls the library from the originating Python thread, even + :mod:`!tkinter` calls the library from the originating Python thread, even if this is different than the thread that created the Tcl interpreter. A global lock ensures only one call occurs at a time. -* While :mod:`tkinter` allows you to create more than one instance of a :class:`Tk` +* While :mod:`!tkinter` allows you to create more than one instance of a :class:`Tk` object (with its own interpreter), all interpreters that are part of the same thread share a common event queue, which gets ugly fast. In practice, don't create more than one instance of :class:`Tk` at a time. Otherwise, it's best to create @@ -548,7 +548,7 @@ A number of special cases exist: or abandon the event loop entirely. If you're doing anything tricky when it comes to events or threads, be aware of these possibilities. -* There are a few select :mod:`tkinter` functions that presently work only when +* There are a few select :mod:`!tkinter` functions that presently work only when called from the thread that created the Tcl interpreter. @@ -636,15 +636,15 @@ The Packer .. index:: single: packing (widgets) The packer is one of Tk's geometry-management mechanisms. Geometry managers -are used to specify the relative positioning of widgets within their container - -their mutual *master*. In contrast to the more cumbersome *placer* (which is +are used to specify the relative positioning of widgets within their container. +In contrast to the more cumbersome *placer* (which is used less commonly, and we do not cover here), the packer takes qualitative relationship specification - *above*, *to the left of*, *filling*, etc - and works everything out to determine the exact placement coordinates for you. -The size of any *master* widget is determined by the size of the "slave widgets" -inside. The packer is used to control where slave widgets appear inside the -master into which they are packed. You can pack widgets into frames, and frames +The size of any container widget is determined by the size of the "content widgets" +inside. The packer is used to control where content widgets appear inside the +container into which they are packed. You can pack widgets into frames, and frames into other frames, in order to achieve the kind of layout you desire. Additionally, the arrangement is dynamically adjusted to accommodate incremental changes to the configuration, once it is packed. @@ -671,7 +671,7 @@ For more extensive information on the packer and the options that it can take, see the man pages and page 183 of John Ousterhout's book. anchor - Anchor type. Denotes where the packer is to place each slave in its parcel. + Anchor type. Denotes where the packer is to place each content in its parcel. expand Boolean, ``0`` or ``1``. @@ -680,10 +680,10 @@ fill Legal values: ``'x'``, ``'y'``, ``'both'``, ``'none'``. ipadx and ipady - A distance - designating internal padding on each side of the slave widget. + A distance - designating internal padding on each side of the content. padx and pady - A distance - designating external padding on each side of the slave widget. + A distance - designating external padding on each side of the content. side Legal values are: ``'left'``, ``'right'``, ``'top'``, ``'bottom'``. @@ -698,11 +698,11 @@ options are ``variable``, ``textvariable``, ``onvalue``, ``offvalue``, and ``value``. This connection works both ways: if the variable changes for any reason, the widget it's connected to will be updated to reflect the new value. -Unfortunately, in the current implementation of :mod:`tkinter` it is not +Unfortunately, in the current implementation of :mod:`!tkinter` it is not possible to hand over an arbitrary Python variable to a widget through a ``variable`` or ``textvariable`` option. The only kinds of variables for which this works are variables that are subclassed from a class called Variable, -defined in :mod:`tkinter`. +defined in :mod:`!tkinter`. There are many useful subclasses of Variable already defined: :class:`StringVar`, :class:`IntVar`, :class:`DoubleVar`, and @@ -750,14 +750,14 @@ The Window Manager In Tk, there is a utility command, ``wm``, for interacting with the window manager. Options to the ``wm`` command allow you to control things like titles, -placement, icon bitmaps, and the like. In :mod:`tkinter`, these commands have +placement, icon bitmaps, and the like. In :mod:`!tkinter`, these commands have been implemented as methods on the :class:`Wm` class. Toplevel widgets are subclassed from the :class:`Wm` class, and so can call the :class:`Wm` methods directly. To get at the toplevel window that contains a given widget, you can often just -refer to the widget's master. Of course if the widget has been packed inside of -a frame, the master won't represent a toplevel window. To get at the toplevel +refer to the widget's :attr:`master`. Of course if the widget has been packed inside of +a frame, the :attr:`!master` won't represent a toplevel window. To get at the toplevel window that contains an arbitrary widget, you can call the :meth:`_root` method. This method begins with an underscore to denote the fact that this function is part of the implementation, and not an interface to Tk functionality. @@ -839,8 +839,7 @@ geometry For example: ``fred["geometry"] = "200x100"``. justify - Legal values are the strings: ``"left"``, ``"center"``, ``"right"``, and - ``"fill"``. + Legal values are the strings: ``"left"``, ``"center"``, and ``"right"``. region This is a string with four space-delimited elements, each of which is a legal @@ -933,7 +932,7 @@ Entry widget, or to particular menu items in a Menu widget. Entry widget indexes (index, view index, etc.) Entry widgets have options that refer to character positions in the text being - displayed. You can use these :mod:`tkinter` functions to access these special + displayed. You can use these :mod:`!tkinter` functions to access these special points in text widgets: Text widget indexes diff --git a/Doc/library/tkinter.scrolledtext.rst b/Doc/library/tkinter.scrolledtext.rst index 763e24929d7..eb30b9c3eac 100644 --- a/Doc/library/tkinter.scrolledtext.rst +++ b/Doc/library/tkinter.scrolledtext.rst @@ -2,16 +2,13 @@ ===================================================== .. module:: tkinter.scrolledtext - :platform: Tk :synopsis: Text widget with a vertical scroll bar. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/tkinter/scrolledtext.py` -------------- -The :mod:`tkinter.scrolledtext` module provides a class of the same name which +The :mod:`!tkinter.scrolledtext` module provides a class of the same name which implements a basic text widget which has a vertical scroll bar configured to do the "right thing." Using the :class:`ScrolledText` class is a lot easier than setting up a text widget and scroll bar directly. diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index 628e9f945ac..e1383e189a3 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -4,20 +4,18 @@ .. module:: tkinter.ttk :synopsis: Tk themed widget set -.. sectionauthor:: Guilherme Polo - **Source code:** :source:`Lib/tkinter/ttk.py` .. index:: single: ttk -------------- -The :mod:`tkinter.ttk` module provides access to the Tk themed widget set, +The :mod:`!tkinter.ttk` module provides access to the Tk themed widget set, introduced in Tk 8.5. It provides additional benefits including anti-aliased font rendering under X11 and window transparency (requiring a composition window manager on X11). -The basic idea for :mod:`tkinter.ttk` is to separate, to the extent possible, +The basic idea for :mod:`!tkinter.ttk` is to separate, to the extent possible, the code implementing a widget's behavior from the code implementing its appearance. @@ -40,7 +38,7 @@ To override the basic Tk widgets, the import should follow the Tk import:: from tkinter import * from tkinter.ttk import * -That code causes several :mod:`tkinter.ttk` widgets (:class:`Button`, +That code causes several :mod:`!tkinter.ttk` widgets (:class:`Button`, :class:`Checkbutton`, :class:`Entry`, :class:`Frame`, :class:`Label`, :class:`LabelFrame`, :class:`Menubutton`, :class:`PanedWindow`, :class:`Radiobutton`, :class:`Scale` and :class:`Scrollbar`) to diff --git a/Doc/library/token.rst b/Doc/library/token.rst index c228006d4c1..3253be96238 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -4,8 +4,6 @@ .. module:: token :synopsis: Constants representing terminal nodes of the parse tree. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/token.py` -------------- @@ -50,8 +48,7 @@ The token constants are: .. data:: NAME - Token value that indicates an :ref:`identifier `. - Note that keywords are also initially tokenized as ``NAME`` tokens. + Token value that indicates an :ref:`identifier or keyword `. .. data:: NUMBER diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index b80917eae66..72fbcaba160 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -4,14 +4,11 @@ .. module:: tokenize :synopsis: Lexical scanner for Python source code. -.. moduleauthor:: Ka Ping Yee -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/tokenize.py` -------------- -The :mod:`tokenize` module provides a lexical scanner for Python source code, +The :mod:`!tokenize` module provides a lexical scanner for Python source code, implemented in Python. The scanner in this module returns comments as tokens as well, making it useful for implementing "pretty-printers", including colorizers for on-screen displays. @@ -31,7 +28,7 @@ type can be determined by checking the ``exact_type`` property on the **undefined** when providing invalid Python code and it can change at any point. -Tokenizing Input +Tokenizing input ---------------- The primary entry point is a :term:`generator`: @@ -78,7 +75,7 @@ The primary entry point is a :term:`generator`: :func:`.tokenize`. It does not yield an :data:`~token.ENCODING` token. All constants from the :mod:`token` module are also exported from -:mod:`tokenize`. +:mod:`!tokenize`. Another function is provided to reverse the tokenization process. This is useful for creating tools that tokenize a script, modify the token stream, and @@ -149,12 +146,12 @@ function it uses to do this is available: .. _tokenize-cli: -Command-Line Usage +Command-line usage ------------------ .. versionadded:: 3.3 -The :mod:`tokenize` module can be executed as a script from the command line. +The :mod:`!tokenize` module can be executed as a script from the command line. It is as simple as: .. code-block:: sh @@ -176,8 +173,12 @@ The following options are accepted: If :file:`filename.py` is specified its contents are tokenized to stdout. Otherwise, tokenization is performed on stdin. +.. versionadded:: next + Output is in color by default and can be + :ref:`controlled using environment variables `. + Examples ------------------- +-------- Example of a script rewriter that transforms float literals into Decimal objects:: @@ -230,7 +231,7 @@ Example of tokenizing from the command line. The script:: will be tokenized to the following output where the first column is the range of the line/column coordinates where the token is found, the second column is -the name of the token, and the final column is the value of the token (if any) +the name of the token, and the final column is the value of the token (if any): .. code-block:: shell-session diff --git a/Doc/library/tomllib.rst b/Doc/library/tomllib.rst index 30d7ff50a1a..55610784362 100644 --- a/Doc/library/tomllib.rst +++ b/Doc/library/tomllib.rst @@ -4,19 +4,28 @@ .. module:: tomllib :synopsis: Parse TOML files. -.. versionadded:: 3.11 - -.. moduleauthor:: Taneli Hukkinen -.. sectionauthor:: Taneli Hukkinen - **Source code:** :source:`Lib/tomllib` -------------- -This module provides an interface for parsing TOML 1.0.0 (Tom's Obvious Minimal +This module provides an interface for parsing TOML 1.1.0 (Tom's Obvious Minimal Language, `https://toml.io `_). This module does not support writing TOML. +.. versionadded:: 3.11 + The module was added with support for TOML 1.0.0. + +.. versionchanged:: 3.15 + Added TOML 1.1.0 support. + See the :ref:`What's New ` for details. + +.. warning:: + + Be cautious when parsing data from untrusted sources. + A malicious TOML string may cause the decoder to consume considerable + CPU and memory resources. + Limiting the size of data to be parsed is recommended. + .. seealso:: The :pypi:`Tomli-W package ` diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index cae94ea08e1..57b8fa29eb3 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -8,7 +8,7 @@ -------------- -The :mod:`trace` module allows you to trace program execution, generate +The :mod:`!trace` module allows you to trace program execution, generate annotated statement coverage listings, print caller/callee relationships and list functions executed during a program run. It can be used in another program or from the command line. @@ -24,7 +24,7 @@ or from the command line. Command-Line Usage ------------------ -The :mod:`trace` module can be invoked from the command line. It can be as +The :mod:`!trace` module can be invoked from the command line. It can be as simple as :: python -m trace --count -C . somefile.py ... @@ -43,13 +43,13 @@ all Python modules imported during the execution into the current directory. Display the version of the module and exit. .. versionadded:: 3.8 - Added ``--module`` option that allows to run an executable module. + Added ``--module`` option that allows running an executable module. Main options ^^^^^^^^^^^^ At least one of the following options must be specified when invoking -:mod:`trace`. The :option:`--listfuncs <-l>` option is mutually exclusive with +:mod:`!trace`. The :option:`--listfuncs <-l>` option is mutually exclusive with the :option:`--trace <-t>` and :option:`--count <-c>` options. When :option:`--listfuncs <-l>` is provided, neither :option:`--count <-c>` nor :option:`--trace <-t>` are accepted, and vice versa. diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst index 2370d927292..0fa70389f1f 100644 --- a/Doc/library/tracemalloc.rst +++ b/Doc/library/tracemalloc.rst @@ -307,7 +307,7 @@ Functions .. function:: get_object_traceback(obj) Get the traceback where the Python object *obj* was allocated. - Return a :class:`Traceback` instance, or ``None`` if the :mod:`tracemalloc` + Return a :class:`Traceback` instance, or ``None`` if the :mod:`!tracemalloc` module is not tracing memory allocations or did not trace the allocation of the object. @@ -318,7 +318,7 @@ Functions Get the maximum number of frames stored in the traceback of a trace. - The :mod:`tracemalloc` module must be tracing memory allocations to + The :mod:`!tracemalloc` module must be tracing memory allocations to get the limit, otherwise an exception is raised. The limit is set by the :func:`start` function. @@ -327,15 +327,15 @@ Functions .. function:: get_traced_memory() Get the current size and peak size of memory blocks traced by the - :mod:`tracemalloc` module as a tuple: ``(current: int, peak: int)``. + :mod:`!tracemalloc` module as a tuple: ``(current: int, peak: int)``. .. function:: reset_peak() - Set the peak size of memory blocks traced by the :mod:`tracemalloc` module + Set the peak size of memory blocks traced by the :mod:`!tracemalloc` module to the current size. - Do nothing if the :mod:`tracemalloc` module is not tracing memory + Do nothing if the :mod:`!tracemalloc` module is not tracing memory allocations. This function only modifies the recorded peak size, and does not modify or @@ -350,14 +350,14 @@ Functions .. function:: get_tracemalloc_memory() - Get the memory usage in bytes of the :mod:`tracemalloc` module used to store + Get the memory usage in bytes of the :mod:`!tracemalloc` module used to store traces of memory blocks. Return an :class:`int`. .. function:: is_tracing() - ``True`` if the :mod:`tracemalloc` module is tracing Python memory + ``True`` if the :mod:`!tracemalloc` module is tracing Python memory allocations, ``False`` otherwise. See also :func:`start` and :func:`stop` functions. @@ -378,8 +378,8 @@ Functions :meth:`Snapshot.compare_to` and :meth:`Snapshot.statistics` methods. Storing more frames increases the memory and CPU overhead of the - :mod:`tracemalloc` module. Use the :func:`get_tracemalloc_memory` function - to measure how much memory is used by the :mod:`tracemalloc` module. + :mod:`!tracemalloc` module. Use the :func:`get_tracemalloc_memory` function + to measure how much memory is used by the :mod:`!tracemalloc` module. The :envvar:`PYTHONTRACEMALLOC` environment variable (``PYTHONTRACEMALLOC=NFRAME``) and the :option:`-X` ``tracemalloc=NFRAME`` @@ -408,12 +408,12 @@ Functions :class:`Snapshot` instance. The snapshot does not include memory blocks allocated before the - :mod:`tracemalloc` module started to trace memory allocations. + :mod:`!tracemalloc` module started to trace memory allocations. Tracebacks of traces are limited to :func:`get_traceback_limit` frames. Use the *nframe* parameter of the :func:`start` function to store more frames. - The :mod:`tracemalloc` module must be tracing memory allocations to take a + The :mod:`!tracemalloc` module must be tracing memory allocations to take a snapshot, see the :func:`start` function. See also the :func:`get_object_traceback` function. @@ -457,7 +457,7 @@ Filter * ``Filter(True, subprocess.__file__)`` only includes traces of the :mod:`subprocess` module * ``Filter(False, tracemalloc.__file__)`` excludes traces of the - :mod:`tracemalloc` module + :mod:`!tracemalloc` module * ``Filter(False, "")`` excludes empty tracebacks @@ -589,7 +589,7 @@ Snapshot If *cumulative* is ``True``, cumulate size and count of memory blocks of all frames of the traceback of a trace, not only the most recent frame. - The cumulative mode can only be used with *key_type* equals to + The cumulative mode can only be used with *key_type* equal to ``'filename'`` and ``'lineno'``. The result is sorted from the biggest to the smallest by: @@ -720,11 +720,10 @@ Traceback When a snapshot is taken, tracebacks of traces are limited to :func:`get_traceback_limit` frames. See the :func:`take_snapshot` function. The original number of frames of the traceback is stored in the - :attr:`Traceback.total_nframe` attribute. That allows to know if a traceback + :attr:`Traceback.total_nframe` attribute. That allows one to know if a traceback has been truncated by the traceback limit. - The :attr:`Trace.traceback` attribute is an instance of :class:`Traceback` - instance. + The :attr:`Trace.traceback` attribute is a :class:`Traceback` instance. .. versionchanged:: 3.7 Frames are now sorted from the oldest to the most recent, instead of most recent to oldest. diff --git a/Doc/library/tty.rst b/Doc/library/tty.rst index 37778bf20bd..9a8e69f09e8 100644 --- a/Doc/library/tty.rst +++ b/Doc/library/tty.rst @@ -2,24 +2,20 @@ ========================================== .. module:: tty - :platform: Unix :synopsis: Utility functions that perform common terminal control operations. -.. moduleauthor:: Steen Lumholt -.. sectionauthor:: Moshe Zadka - **Source code:** :source:`Lib/tty.py` -------------- -The :mod:`tty` module defines functions for putting the tty into cbreak and raw +The :mod:`!tty` module defines functions for putting the tty into cbreak and raw modes. .. availability:: Unix. Because it requires the :mod:`termios` module, it will work only on Unix. -The :mod:`tty` module defines the following functions: +The :mod:`!tty` module defines the following functions: .. function:: cfmakeraw(mode) diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index fea6b57edf0..20c659756fe 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -5,8 +5,6 @@ .. module:: turtle :synopsis: An educational framework for simple graphics applications -.. sectionauthor:: Gregor Lingl - **Source code:** :source:`Lib/turtle.py` .. testsetup:: default @@ -18,6 +16,9 @@ import os os.remove("my_drawing.ps") + # Destroy the turtle window after tests are complete + # Imported via star import in testsetup + bye() -------------- @@ -29,6 +30,8 @@ introduced in Logo `_, developed by Wally Feurzeig, Seymour Papert and Cynthia Solomon in 1967. +.. include:: ../includes/optional-module.rst + Get started =========== @@ -777,13 +780,17 @@ Turtle motion 180.0 -.. function:: dot(size=None, *color) +.. function:: dot() + dot(size) + dot(color, /) + dot(size, color, /) + dot(size, r, g, b, /) :param size: an integer >= 1 (if given) :param color: a colorstring or a numeric color tuple Draw a circular dot with diameter *size*, using *color*. If *size* is - not given, the maximum of pensize+4 and 2*pensize is used. + not given, the maximum of ``pensize+4`` and ``2*pensize`` is used. .. doctest:: @@ -1152,7 +1159,9 @@ Drawing state Color control ~~~~~~~~~~~~~ -.. function:: pencolor(*args) +.. function:: pencolor() + pencolor(color, /) + pencolor(r, g, b, /) Return or set the pencolor. @@ -1161,7 +1170,7 @@ Color control ``pencolor()`` Return the current pencolor as color specification string or as a tuple (see example). May be used as input to another - color/pencolor/fillcolor call. + color/pencolor/fillcolor/bgcolor call. ``pencolor(colorstring)`` Set pencolor to *colorstring*, which is a Tk color specification string, @@ -1201,7 +1210,9 @@ Color control (50.0, 193.0, 143.0) -.. function:: fillcolor(*args) +.. function:: fillcolor() + fillcolor(color, /) + fillcolor(r, g, b, /) Return or set the fillcolor. @@ -1210,7 +1221,7 @@ Color control ``fillcolor()`` Return the current fillcolor as color specification string, possibly in tuple format (see example). May be used as input to another - color/pencolor/fillcolor call. + color/pencolor/fillcolor/bgcolor call. ``fillcolor(colorstring)`` Set fillcolor to *colorstring*, which is a Tk color specification string, @@ -1244,7 +1255,10 @@ Color control (255.0, 255.0, 255.0) -.. function:: color(*args) +.. function:: color() + color(color, /) + color(r, g, b, /) + color(pencolor, fillcolor, /) Return or set pencolor and fillcolor. @@ -1870,13 +1884,32 @@ Most of the examples in this section refer to a TurtleScreen instance called Window control -------------- -.. function:: bgcolor(*args) +.. function:: bgcolor() + bgcolor(color, /) + bgcolor(r, g, b, /) - :param args: a color string or three numbers in the range 0..colormode or a - 3-tuple of such numbers + Return or set the background color of the TurtleScreen. + Four input formats are allowed: - Set or return background color of the TurtleScreen. + ``bgcolor()`` + Return the current background color as color specification string or + as a tuple (see example). May be used as input to another + color/pencolor/fillcolor/bgcolor call. + + ``bgcolor(colorstring)`` + Set the background color to *colorstring*, which is a Tk color + specification string, such as ``"red"``, ``"yellow"``, or ``"#33cc8c"``. + + ``bgcolor((r, g, b))`` + Set the background color to the RGB color represented by the tuple of + *r*, *g*, and *b*. + Each of *r*, *g*, and *b* must be in the range 0..colormode, where + colormode is either 1.0 or 255 (see :func:`colormode`). + + ``bgcolor(r, g, b)`` + Set the background color to the RGB color represented by *r*, *g*, and *b*. Each of + *r*, *g*, and *b* must be in the range 0..colormode. .. doctest:: :skipif: _tkinter is None @@ -2216,7 +2249,7 @@ Settings and special methods Set turtle mode ("standard", "logo" or "world") and perform reset. If mode is not given, current mode is returned. - Mode "standard" is compatible with old :mod:`turtle`. Mode "logo" is + Mode "standard" is compatible with old :mod:`!turtle`. Mode "logo" is compatible with most Logo turtle graphics. Mode "world" uses user-defined "world coordinates". **Attention**: in this mode angles appear distorted if ``x/y`` unit-ratio doesn't equal 1. @@ -2657,7 +2690,7 @@ Screen and Turtle. Python script :file:`{filename}.py`. It is intended to serve as a template for translation of the docstrings into different languages. -If you (or your students) want to use :mod:`turtle` with online help in your +If you (or your students) want to use :mod:`!turtle` with online help in your native language, you have to translate the docstrings and save the resulting file as e.g. :file:`turtle_docstringdict_german.py`. @@ -2720,7 +2753,7 @@ Short explanation of selected entries: auto``. - If you set e.g. ``language = italian`` the docstringdict :file:`turtle_docstringdict_italian.py` will be loaded at import time (if - present on the import path, e.g. in the same directory as :mod:`turtle`). + present on the import path, e.g. in the same directory as :mod:`!turtle`). - The entries *exampleturtle* and *examplescreen* define the names of these objects as they occur in the docstrings. The transformation of method-docstrings to function-docstrings will delete these names from the @@ -2729,7 +2762,7 @@ Short explanation of selected entries: switch ("no subprocess"). This will prevent :func:`exitonclick` to enter the mainloop. -There can be a :file:`turtle.cfg` file in the directory where :mod:`turtle` is +There can be a :file:`turtle.cfg` file in the directory where :mod:`!turtle` is stored and an additional one in the current working directory. The latter will override the settings of the first one. @@ -2738,13 +2771,13 @@ study it as an example and see its effects when running the demos (preferably not from within the demo-viewer). -:mod:`turtledemo` --- Demo scripts -================================== +:mod:`!turtledemo` --- Demo scripts +=================================== .. module:: turtledemo :synopsis: A viewer for example turtle scripts -The :mod:`turtledemo` package includes a set of demo scripts. These +The :mod:`!turtledemo` package includes a set of demo scripts. These scripts can be run and viewed using the supplied demo viewer as follows:: python -m turtledemo @@ -2753,11 +2786,11 @@ Alternatively, you can run the demo scripts individually. For example, :: python -m turtledemo.bytedesign -The :mod:`turtledemo` package directory contains: +The :mod:`!turtledemo` package directory contains: - A demo viewer :file:`__main__.py` which can be used to view the sourcecode of the scripts and run them at the same time. -- Multiple scripts demonstrating different features of the :mod:`turtle` +- Multiple scripts demonstrating different features of the :mod:`!turtle` module. Examples can be accessed via the Examples menu. They can also be run standalone. - A :file:`turtle.cfg` file which serves as an example of how to write @@ -2769,68 +2802,68 @@ The demo scripts are: .. tabularcolumns:: |l|L|L| -+----------------+------------------------------+-----------------------+ -| Name | Description | Features | -+================+==============================+=======================+ -| bytedesign | complex classical | :func:`tracer`, delay,| -| | turtle graphics pattern | :func:`update` | -+----------------+------------------------------+-----------------------+ -| chaos | graphs Verhulst dynamics, | world coordinates | -| | shows that computer's | | -| | computations can generate | | -| | results sometimes against the| | -| | common sense expectations | | -+----------------+------------------------------+-----------------------+ -| clock | analog clock showing time | turtles as clock's | -| | of your computer | hands, ontimer | -+----------------+------------------------------+-----------------------+ -| colormixer | experiment with r, g, b | :func:`ondrag` | -+----------------+------------------------------+-----------------------+ -| forest | 3 breadth-first trees | randomization | -+----------------+------------------------------+-----------------------+ -| fractalcurves | Hilbert & Koch curves | recursion | -+----------------+------------------------------+-----------------------+ -| lindenmayer | ethnomathematics | L-System | -| | (indian kolams) | | -+----------------+------------------------------+-----------------------+ -| minimal_hanoi | Towers of Hanoi | Rectangular Turtles | -| | | as Hanoi discs | -| | | (shape, shapesize) | -+----------------+------------------------------+-----------------------+ -| nim | play the classical nim game | turtles as nimsticks, | -| | with three heaps of sticks | event driven (mouse, | -| | against the computer. | keyboard) | -+----------------+------------------------------+-----------------------+ -| paint | super minimalistic | :func:`onclick` | -| | drawing program | | -+----------------+------------------------------+-----------------------+ -| peace | elementary | turtle: appearance | -| | | and animation | -+----------------+------------------------------+-----------------------+ -| penrose | aperiodic tiling with | :func:`stamp` | -| | kites and darts | | -+----------------+------------------------------+-----------------------+ -| planet_and_moon| simulation of | compound shapes, | -| | gravitational system | :class:`Vec2D` | -+----------------+------------------------------+-----------------------+ -| rosette | a pattern from the wikipedia | :func:`clone`, | -| | article on turtle graphics | :func:`undo` | -+----------------+------------------------------+-----------------------+ -| round_dance | dancing turtles rotating | compound shapes, clone| -| | pairwise in opposite | shapesize, tilt, | -| | direction | get_shapepoly, update | -+----------------+------------------------------+-----------------------+ -| sorting_animate| visual demonstration of | simple alignment, | -| | different sorting methods | randomization | -+----------------+------------------------------+-----------------------+ -| tree | a (graphical) breadth | :func:`clone` | -| | first tree (using generators)| | -+----------------+------------------------------+-----------------------+ -| two_canvases | simple design | turtles on two | -| | | canvases | -+----------------+------------------------------+-----------------------+ -| yinyang | another elementary example | :func:`circle` | -+----------------+------------------------------+-----------------------+ ++------------------------+------------------------------+--------------------------------------+ +| Name | Description | Features | ++========================+==============================+======================================+ +| ``bytedesign`` | complex classical | :func:`tracer`, :func:`delay`, | +| | turtle graphics pattern | :func:`update` | ++------------------------+------------------------------+--------------------------------------+ +| ``chaos`` | graphs Verhulst dynamics, | world coordinates | +| | shows that computer's | | +| | computations can generate | | +| | results sometimes against the| | +| | common sense expectations | | ++------------------------+------------------------------+--------------------------------------+ +| ``clock`` | analog clock showing time | turtles as clock's | +| | of your computer | hands, :func:`ontimer` | ++------------------------+------------------------------+--------------------------------------+ +| ``colormixer`` | experiment with r, g, b | :func:`ondrag` | ++------------------------+------------------------------+--------------------------------------+ +| ``forest`` | 3 breadth-first trees | randomization | ++------------------------+------------------------------+--------------------------------------+ +| ``fractalcurves`` | Hilbert & Koch curves | recursion | ++------------------------+------------------------------+--------------------------------------+ +| ``lindenmayer`` | ethnomathematics | L-System | +| | (indian kolams) | | ++------------------------+------------------------------+--------------------------------------+ +| ``minimal_hanoi`` | Towers of Hanoi | Rectangular Turtles | +| | | as Hanoi discs | +| | | (:func:`shape`, :func:`shapesize`) | ++------------------------+------------------------------+--------------------------------------+ +| ``nim`` | play the classical nim game | turtles as nimsticks, | +| | with three heaps of sticks | event driven (mouse, | +| | against the computer. | keyboard) | ++------------------------+------------------------------+--------------------------------------+ +| ``paint`` | super minimalistic | :func:`onclick` | +| | drawing program | | ++------------------------+------------------------------+--------------------------------------+ +| ``peace`` | elementary | turtle: appearance | +| | | and animation | ++------------------------+------------------------------+--------------------------------------+ +| ``penrose`` | aperiodic tiling with | :func:`stamp` | +| | kites and darts | | ++------------------------+------------------------------+--------------------------------------+ +| ``planet_and_moon`` | simulation of | compound shapes, | +| | gravitational system | :class:`Vec2D` | ++------------------------+------------------------------+--------------------------------------+ +| ``rosette`` | a pattern from the wikipedia | :func:`clone`, | +| | article on turtle graphics | :func:`undo` | ++------------------------+------------------------------+--------------------------------------+ +| ``round_dance`` | dancing turtles rotating | compound shapes, :func:`clone` | +| | pairwise in opposite | :func:`shapesize`, :func:`tilt`, | +| | direction | :func:`get_shapepoly`, :func:`update`| ++------------------------+------------------------------+--------------------------------------+ +| ``sorting_animate`` | visual demonstration of | simple alignment, | +| | different sorting methods | randomization | ++------------------------+------------------------------+--------------------------------------+ +| ``tree`` | a (graphical) breadth | :func:`clone` | +| | first tree (using generators)| | ++------------------------+------------------------------+--------------------------------------+ +| ``two_canvases`` | simple design | turtles on two | +| | | canvases | ++------------------------+------------------------------+--------------------------------------+ +| ``yinyang`` | another elementary example | :func:`circle` | ++------------------------+------------------------------+--------------------------------------+ Have fun! diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 207024a7619..74898baa521 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -338,11 +338,23 @@ Standard names are defined for the following types: The type of frame locals proxy objects, as found on the :attr:`frame.f_locals` attribute. - .. versionadded:: next + .. versionadded:: 3.15 .. seealso:: :pep:`667` +.. data:: LazyImportType + + The type of lazy import proxy objects. These objects are created when a + module is lazily imported and serve as placeholders until the module is + actually accessed. This type can be used to detect lazy imports + programmatically. + + .. versionadded:: 3.15 + + .. seealso:: :pep:`810` + + .. data:: GetSetDescriptorType The type of objects defined in extension modules with ``PyGetSetDef``, such diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 022c76b084c..17cf57dd00b 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -813,7 +813,7 @@ For example, this conforms to :pep:`484`:: def __len__(self) -> int: ... def __iter__(self) -> Iterator[int]: ... -:pep:`544` allows to solve this problem by allowing users to write +:pep:`544` solves this problem by allowing users to write the above code without explicit base classes in the class definition, allowing ``Bucket`` to be implicitly considered a subtype of both ``Sized`` and ``Iterable[int]`` by static type checkers. This is known as @@ -1174,7 +1174,8 @@ These can be used as types in annotations. They all support subscription using or transforms parameters of another callable. Usage is in the form ``Concatenate[Arg1Type, Arg2Type, ..., ParamSpecVariable]``. ``Concatenate`` - is currently only valid when used as the first argument to a :ref:`Callable `. + is valid when used in :ref:`Callable ` type hints + and when instantiating user-defined generic classes with :class:`ParamSpec` parameters. The last parameter to ``Concatenate`` must be a :class:`ParamSpec` or ellipsis (``...``). @@ -1390,7 +1391,7 @@ These can be used as types in annotations. They all support subscription using Using ``Annotated[T, x]`` as an annotation still allows for static typechecking of ``T``, as type checkers will simply ignore the metadata ``x``. In this way, ``Annotated`` differs from the - :func:`@no_type_check ` decorator, which can also be used for + :deco:`no_type_check` decorator, which can also be used for adding annotations outside the scope of the typing system, but completely disables typechecking for a function or class. @@ -1524,6 +1525,35 @@ These can be used as types in annotations. They all support subscription using .. versionadded:: 3.9 +.. data:: TypeForm + + A special form representing the value that results from evaluating a + type expression. + + This value encodes the information supplied in the type expression, and + it represents the type described by that type expression. + + When used in a type expression, ``TypeForm`` describes a set of type form + objects. It accepts a single type argument, which must be a valid type + expression. ``TypeForm[T]`` describes the set of all type form objects that + represent the type ``T`` or types assignable to ``T``. + + ``TypeForm(obj)`` simply returns ``obj`` unchanged. This is useful for + explicitly marking a value as a type form for static type checkers. + + Example:: + + from typing import Any, TypeForm + + def cast[T](typ: TypeForm[T], value: Any) -> T: ... + + reveal_type(cast(int, "x")) # Revealed type is "int" + + See :pep:`747` for details. + + .. versionadded:: 3.15 + + .. data:: TypeIs Special typing construct for marking user-defined type predicate functions. @@ -1951,7 +1981,7 @@ without the dedicated syntax, as documented below. .. _typevartuple: -.. class:: TypeVarTuple(name, *, default=typing.NoDefault) +.. class:: TypeVarTuple(name, *, bound=None, covariant=False, contravariant=False, infer_variance=False, default=typing.NoDefault) Type variable tuple. A specialized form of :ref:`type variable ` that enables *variadic* generics. @@ -2061,6 +2091,24 @@ without the dedicated syntax, as documented below. The name of the type variable tuple. + .. attribute:: __covariant__ + + Whether the type variable tuple has been explicitly marked as covariant. + + .. versionadded:: 3.15 + + .. attribute:: __contravariant__ + + Whether the type variable tuple has been explicitly marked as contravariant. + + .. versionadded:: 3.15 + + .. attribute:: __infer_variance__ + + Whether the type variable tuple's variance should be inferred by type checkers. + + .. versionadded:: 3.15 + .. attribute:: __default__ The default value of the type variable tuple, or :data:`typing.NoDefault` if it @@ -2087,6 +2135,11 @@ without the dedicated syntax, as documented below. .. versionadded:: 3.13 + Type variable tuples created with ``covariant=True`` or + ``contravariant=True`` can be used to declare covariant or contravariant + generic types. The ``bound`` argument is also accepted, similar to + :class:`TypeVar`, but its actual semantics are yet to be decided. + .. versionadded:: 3.11 .. versionchanged:: 3.12 @@ -2098,6 +2151,11 @@ without the dedicated syntax, as documented below. Support for default values was added. + .. versionchanged:: 3.15 + + Added support for the ``bound``, ``covariant``, ``contravariant``, and + ``infer_variance`` parameters. + .. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False, default=typing.NoDefault) Parameter specification variable. A specialized version of @@ -2167,6 +2225,20 @@ without the dedicated syntax, as documented below. The name of the parameter specification. + .. attribute:: __covariant__ + + Whether the parameter specification has been explicitly marked as covariant. + + .. attribute:: __contravariant__ + + Whether the parameter specification has been explicitly marked as contravariant. + + .. attribute:: __infer_variance__ + + Whether the parameter specification's variance should be inferred by type checkers. + + .. versionadded:: 3.12 + .. attribute:: __default__ The default value of the parameter specification, or :data:`typing.NoDefault` if it @@ -2243,7 +2315,7 @@ without the dedicated syntax, as documented below. .. versionadded:: 3.10 -.. class:: TypeAliasType(name, value, *, type_params=()) +.. class:: TypeAliasType(name, value, *, type_params=(), qualname=None) The type of type aliases created through the :keyword:`type` statement. @@ -2267,14 +2339,34 @@ without the dedicated syntax, as documented below. >>> Alias.__name__ 'Alias' + .. attribute:: __qualname__ + + The :term:`qualified name` of the type alias: + + .. doctest:: + + >>> class Class: + ... type Alias = int + ... + >>> Class.Alias.__qualname__ + 'Class.Alias' + + .. versionadded:: 3.15 + .. attribute:: __module__ - The module in which the type alias was defined:: + The name of the module in which the type alias was defined:: >>> type Alias = int >>> Alias.__module__ '__main__' + This attribute is writable. + + .. versionchanged:: 3.15 + + The attribute is now writable. + .. attribute:: __type_params__ The type parameters of the type alias, or an empty tuple if the alias is @@ -2428,6 +2520,10 @@ types. Removed the ``_field_types`` attribute in favor of the more standard ``__annotations__`` attribute which has the same information. + .. versionchanged:: 3.9 + ``NamedTuple`` is now a function rather than a class. + It can still be used as a class base, as described above. + .. versionchanged:: 3.11 Added support for generic namedtuples. @@ -2435,19 +2531,6 @@ types. Using :func:`super` (and the ``__class__`` :term:`closure variable`) in methods of ``NamedTuple`` subclasses is unsupported and causes a :class:`TypeError`. - .. deprecated-removed:: 3.13 3.15 - The undocumented keyword argument syntax for creating NamedTuple classes - (``NT = NamedTuple("NT", x=int)``) is deprecated, and will be disallowed - in 3.15. Use the class-based syntax or the functional syntax instead. - - .. deprecated-removed:: 3.13 3.15 - When using the functional syntax to create a NamedTuple class, failing to - pass a value to the 'fields' parameter (``NT = NamedTuple("NT")``) is - deprecated. Passing ``None`` to the 'fields' parameter - (``NT = NamedTuple("NT", None)``) is also deprecated. Both will be - disallowed in Python 3.15. To create a NamedTuple class with 0 fields, - use ``class NT(NamedTuple): pass`` or ``NT = NamedTuple("NT", [])``. - .. class:: NewType(name, tp) Helper class to create low-overhead :ref:`distinct types `. @@ -2462,7 +2545,7 @@ types. .. attribute:: __module__ - The module in which the new type is defined. + The name of the module in which the new type is defined. .. attribute:: __name__ @@ -2522,6 +2605,12 @@ types. .. versionadded:: 3.8 + .. deprecated-removed:: 3.15 3.20 + It is deprecated to call :func:`isinstance` and :func:`issubclass` checks on + protocol classes that were not explicitly decorated with :func:`!runtime_checkable` + but that inherit from a runtime-checkable protocol class. This will throw + a :exc:`TypeError` in Python 3.20. + .. decorator:: runtime_checkable Mark a protocol class as a runtime protocol. @@ -2543,6 +2632,18 @@ types. import threading assert isinstance(threading.Thread(name='Bob'), Named) + Runtime checkability of protocols is not inherited. A subclass of a runtime-checkable protocol + is only runtime-checkable if it is explicitly marked as such, regardless of class hierarchy:: + + @runtime_checkable + class Iterable(Protocol): + def __iter__(self): ... + + # Without @runtime_checkable, Reversible would no longer be runtime-checkable. + @runtime_checkable + class Reversible(Iterable, Protocol): + def __reversed__(self): ... + This decorator raises :exc:`TypeError` when applied to a non-protocol class. .. note:: @@ -2583,11 +2684,16 @@ types. protocol. See :ref:`What's new in Python 3.12 ` for more details. + .. deprecated-removed:: 3.15 3.20 + It is deprecated to call :func:`isinstance` and :func:`issubclass` checks on + protocol classes that were not explicitly decorated with :func:`!runtime_checkable` + but that inherit from a runtime-checkable protocol class. This will throw + a :exc:`TypeError` in Python 3.20. .. class:: TypedDict(dict) Special construct to add type hints to a dictionary. - At runtime it is a plain :class:`dict`. + At runtime ":class:`!TypedDict` instances" are simply :class:`dicts `. ``TypedDict`` declares a dictionary type that expects all of its instances to have a certain set of keys, where each key is @@ -2810,6 +2916,10 @@ types. .. versionadded:: 3.8 + .. versionchanged:: 3.9 + ``TypedDict`` is now a function rather than a class. + It can still be used as a class base, as described above. + .. versionchanged:: 3.11 Added support for marking individual keys as :data:`Required` or :data:`NotRequired`. See :pep:`655`. @@ -2823,19 +2933,12 @@ types. .. versionchanged:: 3.13 Support for the :data:`ReadOnly` qualifier was added. - .. deprecated-removed:: 3.13 3.15 - When using the functional syntax to create a TypedDict class, failing to - pass a value to the 'fields' parameter (``TD = TypedDict("TD")``) is - deprecated. Passing ``None`` to the 'fields' parameter - (``TD = TypedDict("TD", None)``) is also deprecated. Both will be - disallowed in Python 3.15. To create a TypedDict class with 0 fields, - use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. Protocols --------- The following protocols are provided by the :mod:`!typing` module. All are decorated -with :func:`@runtime_checkable `. +with :deco:`runtime_checkable`. .. class:: SupportsAbs @@ -2875,8 +2978,8 @@ ABCs and Protocols for working with I/O --------------------------------------- .. class:: IO[AnyStr] - TextIO[AnyStr] - BinaryIO[AnyStr] + TextIO + BinaryIO Generic class ``IO[AnyStr]`` and its subclasses ``TextIO(IO[str])`` and ``BinaryIO(IO[bytes])`` @@ -3016,7 +3119,7 @@ Functions and decorators The presence of ``@dataclass_transform()`` tells a static type checker that the decorated object performs runtime "magic" that transforms a class in a similar way to - :func:`@dataclasses.dataclass `. + :deco:`dataclasses.dataclass`. Example usage with a decorator function: @@ -3054,14 +3157,14 @@ Functions and decorators The ``CustomerModel`` classes defined above will be treated by type checkers similarly to classes created with - :func:`@dataclasses.dataclass `. + :deco:`dataclasses.dataclass`. For example, type checkers will assume these classes have ``__init__`` methods that accept ``id`` and ``name``. The decorated class, metaclass, or function may accept the following bool arguments which type checkers will assume have the same effect as they would have on the - :func:`@dataclasses.dataclass` decorator: ``init``, + :deco:`dataclasses.dataclass` decorator: ``init``, ``eq``, ``order``, ``unsafe_hash``, ``frozen``, ``match_args``, ``kw_only``, and ``slots``. It must be possible for the value of these arguments (``True`` or ``False``) to be statically evaluated. @@ -3189,12 +3292,12 @@ Functions and decorators .. function:: get_overloads(func) - Return a sequence of :func:`@overload `-decorated definitions for + Return a sequence of :deco:`overload`-decorated definitions for *func*. *func* is the function object for the implementation of the overloaded function. For example, given the definition of ``process`` in - the documentation for :func:`@overload `, + the documentation for :deco:`overload`, ``get_overloads(process)`` will return a sequence of three function objects for the three defined overloads. If called on a function with no overloads, ``get_overloads()`` returns an empty sequence. @@ -3264,17 +3367,6 @@ Functions and decorators ``@no_type_check`` mutates the decorated object in place. -.. decorator:: no_type_check_decorator - - Decorator to give another decorator the :func:`no_type_check` effect. - - This wraps the decorator with something that wraps the decorated - function in :func:`no_type_check`. - - .. deprecated-removed:: 3.13 3.15 - No type checker ever added support for ``@no_type_check_decorator``. It - is therefore deprecated, and will be removed in Python 3.15. - .. decorator:: override Decorator to indicate that a method in a subclass is intended to override a @@ -3315,6 +3407,36 @@ Functions and decorators .. versionadded:: 3.12 +.. decorator:: disjoint_base + + Decorator to mark a class as a disjoint base. + + Type checkers do not allow child classes of a disjoint base ``C`` to + inherit from other disjoint bases that are not parent or child classes of ``C``. + + For example:: + + @disjoint_base + class Disjoint1: pass + + @disjoint_base + class Disjoint2: pass + + class Disjoint3(Disjoint1, Disjoint2): pass # Type checker error + + Type checkers can use knowledge of disjoint bases to detect unreachable code + and determine when two types can overlap. + + The corresponding runtime concept is a solid base (see :ref:`multiple-inheritance`). + Classes that are solid bases at runtime can be marked with ``@disjoint_base`` in stub files. + Users may also mark other classes as disjoint bases to indicate to type checkers that + multiple inheritance with other disjoint bases should not be allowed. + + Note that the concept of a solid base is a CPython implementation + detail, and the exact set of standard library classes that are + disjoint bases at runtime may change in future versions of Python. + + .. versionadded:: next .. decorator:: type_check_only @@ -3337,13 +3459,13 @@ Functions and decorators Introspection helpers --------------------- -.. function:: get_type_hints(obj, globalns=None, localns=None, include_extras=False) +.. function:: get_type_hints(obj, globalns=None, localns=None, include_extras=False, *, format=Format.VALUE) - Return a dictionary containing type hints for a function, method, module - or class object. + Return a dictionary containing type hints for a function, method, module, + class object, or other callable object. - This is often the same as ``obj.__annotations__``, but this function makes - the following changes to the annotations dictionary: + This is often the same as :func:`annotationlib.get_annotations`, but this + function makes the following changes to the annotations dictionary: * Forward references encoded as string literals or :class:`ForwardRef` objects are handled by evaluating them in *globalns*, *localns*, and @@ -3351,22 +3473,21 @@ Introspection helpers If *globalns* or *localns* is not given, appropriate namespace dictionaries are inferred from *obj*. * ``None`` is replaced with :class:`types.NoneType`. - * If :func:`@no_type_check ` has been applied to *obj*, an + * If :deco:`no_type_check` has been applied to *obj*, an empty dictionary is returned. * If *obj* is a class ``C``, the function returns a dictionary that merges annotations from ``C``'s base classes with those on ``C`` directly. This is done by traversing :attr:`C.__mro__ ` and iteratively combining - ``__annotations__`` dictionaries. Annotations on classes appearing - earlier in the :term:`method resolution order` always take precedence over - annotations on classes appearing later in the method resolution order. - * The function recursively replaces all occurrences of ``Annotated[T, ...]`` + :term:`annotations ` of each base class. Annotations + on classes appearing earlier in the :term:`method resolution order` always + take precedence over annotations on classes appearing later in the method + resolution order. + * The function recursively replaces all occurrences of + ``Annotated[T, ...]``, ``Required[T]``, ``NotRequired[T]``, and ``ReadOnly[T]`` with ``T``, unless *include_extras* is set to ``True`` (see :class:`Annotated` for more information). - See also :func:`annotationlib.get_annotations`, a lower-level function that - returns annotations more directly. - .. caution:: This function may execute arbitrary code contained in annotations. @@ -3374,11 +3495,19 @@ Introspection helpers .. note:: - If any forward references in the annotations of *obj* are not resolvable - or are not valid Python code, this function will raise an exception - such as :exc:`NameError`. For example, this can happen with imported - :ref:`type aliases ` that include forward references, - or with names imported under :data:`if TYPE_CHECKING `. + If :attr:`Format.VALUE ` is used and any + forward references in the annotations of *obj* are not resolvable, a + :exc:`NameError` exception is raised. For example, this can happen + with names imported under :data:`if TYPE_CHECKING `. + More generally, any kind of exception can be raised if an annotation + contains invalid Python code. + + .. note:: + + Calling :func:`get_type_hints` on an instance is not supported. + To retrieve annotations for an instance, call + :func:`get_type_hints` on the instance's class instead + (for example, ``get_type_hints(type(obj))``). .. versionchanged:: 3.9 Added ``include_extras`` parameter as part of :pep:`593`. @@ -3389,6 +3518,15 @@ Introspection helpers if a default value equal to ``None`` was set. Now the annotation is returned unchanged. + .. versionchanged:: 3.14 + Added the ``format`` parameter. See the documentation on + :func:`annotationlib.get_annotations` for more information. + + .. versionchanged:: 3.14 + Calling :func:`get_type_hints` on instances is no longer supported. + Some instances were accepted in earlier versions as an undocumented + implementation detail. + .. function:: get_origin(tp) Get the unsubscripted version of a type: for a typing object of the form @@ -3741,7 +3879,7 @@ Aliases to other concrete types Match Deprecated aliases corresponding to the return types from - :func:`re.compile` and :func:`re.match`. + :func:`re.compile` and :func:`re.search`. These types (and the corresponding functions) are generic over :data:`AnyStr`. ``Pattern`` can be specialised as ``Pattern[str]`` or @@ -3788,6 +3926,28 @@ Aliases to container ABCs in :mod:`collections.abc` :class:`collections.abc.Set` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. +.. class:: ByteString(Sequence[int]) + + Deprecated alias to :class:`collections.abc.ByteString`. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the :ref:`buffer protocol ` at runtime. For use in + type annotations, either use :class:`~collections.abc.Buffer` or a union + that explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was an + instance of :class:`!ByteString` never actually told you anything useful + about the object. Other common buffer types such as :class:`memoryview` were + also never understood as subtypes of :class:`!ByteString` (either at runtime + or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. + + .. deprecated-removed:: 3.9 3.17 + .. class:: Collection(Sized, Iterable[T_co], Container[T_co]) Deprecated alias to :class:`collections.abc.Collection`. @@ -4081,6 +4241,10 @@ convenience. This is subject to change, and not all deprecations are listed. - 3.9 - Undecided (see :ref:`deprecated-aliases` for more information) - :pep:`585` + * - :class:`typing.ByteString` + - 3.9 + - 3.17 + - :gh:`91896` * - :data:`typing.Text` - 3.11 - Undecided @@ -4093,10 +4257,6 @@ convenience. This is subject to change, and not all deprecations are listed. - 3.12 - Undecided - :pep:`695` - * - :func:`@typing.no_type_check_decorator ` - - 3.13 - - 3.15 - - :gh:`106309` * - :data:`typing.AnyStr` - 3.13 - 3.18 diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index 76f9a038181..f5c11fd849f 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -4,10 +4,6 @@ .. module:: unicodedata :synopsis: Access the Unicode Database. -.. moduleauthor:: Marc-André Lemburg -.. sectionauthor:: Marc-André Lemburg -.. sectionauthor:: Martin v. Löwis - .. index:: single: Unicode single: character @@ -17,8 +13,8 @@ This module provides access to the Unicode Character Database (UCD) which defines character properties for all Unicode characters. The data contained in -this database is compiled from the `UCD version 16.0.0 -`_. +this database is compiled from the `UCD version 17.0.0 +`_. The module uses the same names and symbols as defined by Unicode Standard Annex #44, `"Unicode Character Database" @@ -31,7 +27,7 @@ following functions: this module. -.. function:: lookup(name) +.. function:: lookup(name, /) Look up character by name. If a character with the given name is found, return the corresponding character. If not found, :exc:`KeyError` is raised. @@ -94,7 +90,7 @@ following functions: 0.5 -.. function:: category(chr) +.. function:: category(chr, /) Returns the general category assigned to the character *chr* as string. General category names consist of two letters. @@ -106,7 +102,7 @@ following functions: 'Lu' -.. function:: bidirectional(chr) +.. function:: bidirectional(chr, /) Returns the bidirectional class assigned to the character *chr* as string. If no such value is defined, an empty string is returned. @@ -118,23 +114,35 @@ following functions: 'AN' -.. function:: combining(chr) +.. function:: combining(chr, /) Returns the canonical combining class assigned to the character *chr* as integer. Returns ``0`` if no combining class is defined. See the `Canonical Combining Class Values section of the Unicode Character - Database `_ + Database `_ for more information. -.. function:: east_asian_width(chr) +.. function:: east_asian_width(chr, /) Returns the east asian width assigned to the character *chr* as string. For a list of widths and or more information, see the `Unicode Standard Annex #11 `_. -.. function:: mirrored(chr) +.. function:: block(chr, /) + + Returns the `block + `_ + assigned to the character *chr*. For example:: + + >>> unicodedata.block('S') + 'Basic Latin' + + .. versionadded:: 3.15 + + +.. function:: mirrored(chr, /) Returns the mirrored property assigned to the character *chr* as integer. Returns ``1`` if the character has been identified as a "mirrored" @@ -144,7 +152,37 @@ following functions: 1 -.. function:: decomposition(chr) +.. function:: isxidstart(chr, /) + + Return ``True`` if *chr* is a valid identifier start per the + `Unicode Standard Annex #31 `_, + that is, it has the ``XID_Start`` property. Return ``False`` otherwise. + For example:: + + >>> unicodedata.isxidstart('S') + True + >>> unicodedata.isxidstart('0') + False + + .. versionadded:: 3.15 + + +.. function:: isxidcontinue(chr, /) + + Return ``True`` if *chr* is a valid identifier character per the + `Unicode Standard Annex #31 `_, + that is, it has the ``XID_Continue`` property. Return ``False`` otherwise. + For example:: + + >>> unicodedata.isxidcontinue('S') + True + >>> unicodedata.isxidcontinue(' ') + False + + .. versionadded:: 3.15 + + +.. function:: decomposition(chr, /) Returns the character decomposition mapping assigned to the character *chr* as string. An empty string is returned in case no such mapping is @@ -154,7 +192,29 @@ following functions: '0041 0303' -.. function:: normalize(form, unistr) +.. function:: grapheme_cluster_break(chr, /) + + Returns the Grapheme_Cluster_Break property assigned to the character. + + .. versionadded:: 3.15 + + +.. function:: indic_conjunct_break(chr, /) + + Returns the Indic_Conjunct_Break property assigned to the character. + + .. versionadded:: 3.15 + + +.. function:: extended_pictographic(chr, /) + + Returns ``True`` if the character has the Extended_Pictographic property, + ``False`` otherwise. + + .. versionadded:: 3.15 + + +.. function:: normalize(form, unistr, /) Return the normal form *form* for the Unicode string *unistr*. Valid values for *form* are 'NFC', 'NFKC', 'NFD', and 'NFKD'. @@ -187,7 +247,7 @@ following functions: doesn't, they may not compare equal. -.. function:: is_normalized(form, unistr) +.. function:: is_normalized(form, unistr, /) Return whether the Unicode string *unistr* is in the normal form *form*. Valid values for *form* are 'NFC', 'NFKC', 'NFD', and 'NFKD'. @@ -195,6 +255,24 @@ following functions: .. versionadded:: 3.8 +.. function:: iter_graphemes(unistr, start=0, end=sys.maxsize, /) + + Returns an iterator to iterate over grapheme clusters. + With optional *start*, iteration begins at that position. + With optional *end*, iteration stops at that position. + + Converting an emitted item to string returns a substring corresponding to + the grapheme cluster. + Its ``start`` and ``end`` attributes denote the start and end of + the grapheme cluster. + + It uses extended grapheme cluster rules defined by Unicode + Standard Annex #29, `"Unicode Text Segmentation" + `_. + + .. versionadded:: 3.15 + + In addition, the module exposes the following constant: .. data:: unidata_version @@ -204,13 +282,13 @@ In addition, the module exposes the following constant: .. data:: ucd_3_2_0 - This is an object that has the same methods as the entire module, but uses the + This is an object that has most of the methods of the entire module, but uses the Unicode database version 3.2 instead, for applications that require this specific version of the Unicode database (such as IDNA). .. rubric:: Footnotes -.. [#] https://www.unicode.org/Public/16.0.0/ucd/NameAliases.txt +.. [#] https://www.unicode.org/Public/17.0.0/ucd/NameAliases.txt -.. [#] https://www.unicode.org/Public/16.0.0/ucd/NamedSequences.txt +.. [#] https://www.unicode.org/Public/17.0.0/ucd/NamedSequences.txt diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 00cc9bfc0a5..b8aa04b9ab2 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -1,7 +1,6 @@ :mod:`!unittest.mock` --- getting started ========================================= -.. moduleauthor:: Michael Foord .. currentmodule:: unittest.mock .. versionadded:: 3.3 @@ -26,7 +25,7 @@ Using Mock ---------- -Mock Patching Methods +Mock patching methods ~~~~~~~~~~~~~~~~~~~~~ Common uses for :class:`Mock` objects include: @@ -72,7 +71,7 @@ the ``something`` method: -Mock for Method Calls on an Object +Mock for method calls on an object ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In the last example we patched a method directly on an object to check that it @@ -102,7 +101,7 @@ accessing it in the test will create it, but :meth:`~Mock.assert_called_with` will raise a failure exception. -Mocking Classes +Mocking classes ~~~~~~~~~~~~~~~ A common use case is to mock out classes instantiated by your code under test. @@ -140,7 +139,7 @@ name is also propagated to attributes or methods of the mock: -Tracking all Calls +Tracking all calls ~~~~~~~~~~~~~~~~~~ Often you want to track more than a single call to a method. The @@ -177,7 +176,7 @@ possible to track nested calls where the parameters used to create ancestors are True -Setting Return Values and Attributes +Setting return values and attributes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Setting the return values on a mock object is trivially easy: @@ -318,7 +317,7 @@ return an async function. >>> mock_instance.__aexit__.assert_awaited_once() -Creating a Mock from an Existing Object +Creating a mock from an existing object ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ One problem with over use of mocking is that it couples your tests to the @@ -385,7 +384,7 @@ contents per file stored in a dictionary:: assert file2.read() == "default" -Patch Decorators +Patch decorators ---------------- .. note:: @@ -519,7 +518,7 @@ decorator individually to every method whose name starts with "test". .. _further-examples: -Further Examples +Further examples ---------------- @@ -600,13 +599,13 @@ this list of calls for us:: Partial mocking ~~~~~~~~~~~~~~~ -In some tests I wanted to mock out a call to :meth:`datetime.date.today` -to return a known date, but I didn't want to prevent the code under test from -creating new date objects. Unfortunately :class:`datetime.date` is written in C, and -so I couldn't just monkey-patch out the static :meth:`datetime.date.today` method. +For some tests, you may want to mock out a call to :meth:`datetime.date.today` +to return a known date, but don't want to prevent the code under test from +creating new date objects. Unfortunately :class:`datetime.date` is written in C, +so you cannot just monkey-patch out the static :meth:`datetime.date.today` method. -I found a simple way of doing this that involved effectively wrapping the date -class with a mock, but passing through calls to the constructor to the real +Instead, you can effectively wrap the date +class with a mock, while passing through calls to the constructor to the real class (and returning real instances). The :func:`patch decorator ` is used here to @@ -615,13 +614,13 @@ attribute on the mock date class is then set to a lambda function that returns a real date. When the mock date class is called a real date will be constructed and returned by ``side_effect``. :: - >>> from datetime import date + >>> import datetime as dt >>> with patch('mymodule.date') as mock_date: - ... mock_date.today.return_value = date(2010, 10, 8) - ... mock_date.side_effect = lambda *args, **kw: date(*args, **kw) + ... mock_date.today.return_value = dt.date(2010, 10, 8) + ... mock_date.side_effect = lambda *args, **kw: dt.date(*args, **kw) ... - ... assert mymodule.date.today() == date(2010, 10, 8) - ... assert mymodule.date(2009, 6, 8) == date(2009, 6, 8) + ... assert mymodule.date.today() == dt.date(2010, 10, 8) + ... assert mymodule.date(2009, 6, 8) == dt.date(2009, 6, 8) Note that we don't patch :class:`datetime.date` globally, we patch ``date`` in the module that *uses* it. See :ref:`where to patch `. @@ -639,7 +638,7 @@ is discussed in `this blog entry `_. -Mocking a Generator Method +Mocking a generator method ~~~~~~~~~~~~~~~~~~~~~~~~~~ A Python generator is a function or method that uses the :keyword:`yield` statement @@ -740,19 +739,18 @@ exception is raised in the setUp then tearDown is not called. >>> MyTest('test_foo').run() -Mocking Unbound Methods +Mocking unbound methods ~~~~~~~~~~~~~~~~~~~~~~~ -Whilst writing tests today I needed to patch an *unbound method* (patching the -method on the class rather than on the instance). I needed self to be passed -in as the first argument because I want to make asserts about which objects -were calling this particular method. The issue is that you can't patch with a -mock for this, because if you replace an unbound method with a mock it doesn't -become a bound method when fetched from the instance, and so it doesn't get -self passed in. The workaround is to patch the unbound method with a real -function instead. The :func:`patch` decorator makes it so simple to -patch out methods with a mock that having to create a real function becomes a -nuisance. +Sometimes a test needs to patch an *unbound method*, which means patching the +method on the class rather than on the instance. In order to make assertions +about which objects were calling this particular method, you need to pass +``self`` as the first argument. The issue is that you can't patch with a mock for +this, because if you replace an unbound method with a mock it doesn't become +a bound method when fetched from the instance, and so it doesn't get ``self`` +passed in. The workaround is to patch the unbound method with a real function +instead. The :func:`patch` decorator makes it so simple to patch out methods +with a mock that having to create a real function becomes a nuisance. If you pass ``autospec=True`` to patch then it does the patching with a *real* function object. This function object has the same signature as the one @@ -760,8 +758,8 @@ it is replacing, but delegates to a mock under the hood. You still get your mock auto-created in exactly the same way as before. What it means though, is that if you use it to patch out an unbound method on a class the mocked function will be turned into a bound method if it is fetched from an instance. -It will have ``self`` passed in as the first argument, which is exactly what I -wanted: +It will have ``self`` passed in as the first argument, which is exactly what +was needed: >>> class Foo: ... def foo(self): @@ -864,9 +862,9 @@ Here's one solution that uses the :attr:`~Mock.side_effect` functionality. If you provide a ``side_effect`` function for a mock then ``side_effect`` will be called with the same args as the mock. This gives us an opportunity to copy the arguments and store them for later assertions. In this -example I'm using *another* mock to store the arguments so that I can use the +example we're using *another* mock to store the arguments so that we can use the mock methods for doing the assertion. Again a helper function sets this up for -me. :: +us. :: >>> from copy import deepcopy >>> from unittest.mock import Mock, patch, DEFAULT @@ -939,7 +937,7 @@ and the ``return_value`` will use your subclass automatically. That means all children of a ``CopyingMock`` will also have the type ``CopyingMock``. -Nesting Patches +Nesting patches ~~~~~~~~~~~~~~~ Using patch as a context manager is nice, but if you do multiple patches you diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 91f90a0726a..2ff1015af7a 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -4,7 +4,6 @@ .. module:: unittest.mock :synopsis: Mock object library. -.. moduleauthor:: Michael Foord .. currentmodule:: unittest.mock .. versionadded:: 3.3 @@ -13,11 +12,11 @@ -------------- -:mod:`unittest.mock` is a library for testing in Python. It allows you to +:mod:`!unittest.mock` is a library for testing in Python. It allows you to replace parts of your system under test with mock objects and make assertions about how they have been used. -:mod:`unittest.mock` provides a core :class:`Mock` class removing the need to +:mod:`!unittest.mock` provides a core :class:`Mock` class removing the need to create a host of stubs throughout your test suite. After performing an action, you can make assertions about which methods / attributes were used and arguments they were called with. You can also specify return values and @@ -33,7 +32,7 @@ Mock is designed for use with :mod:`unittest` and is based on the 'action -> assertion' pattern instead of 'record -> replay' used by many mocking frameworks. -There is a backport of :mod:`unittest.mock` for earlier versions of Python, +There is a backport of :mod:`!unittest.mock` for earlier versions of Python, available as :pypi:`mock` on PyPI. @@ -2638,7 +2637,7 @@ unit tests. Testing everything in isolation is all fine and dandy, but if you don't test how your units are "wired together" there is still lots of room for bugs that tests might have caught. -:mod:`unittest.mock` already provides a feature to help with this, called speccing. If you +:mod:`!unittest.mock` already provides a feature to help with this, called speccing. If you use a class or instance as the :attr:`!spec` for a mock then you can only access attributes on the mock that exist on the real class: diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index b5331e0676d..6e0df0648fb 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -4,11 +4,6 @@ .. module:: unittest :synopsis: Unit testing framework for Python. -.. moduleauthor:: Steve Purcell -.. sectionauthor:: Steve Purcell -.. sectionauthor:: Fred L. Drake, Jr. -.. sectionauthor:: Raymond Hettinger - **Source code:** :source:`Lib/unittest/__init__.py` -------------- @@ -16,13 +11,13 @@ (If you are already familiar with the basic concepts of testing, you might want to skip to :ref:`the list of assert methods `.) -The :mod:`unittest` unit testing framework was originally inspired by JUnit +The :mod:`!unittest` unit testing framework was originally inspired by JUnit and has a similar flavor as major unit testing frameworks in other languages. It supports test automation, sharing of setup and shutdown code for tests, aggregation of tests into collections, and independence of the tests from the reporting framework. -To achieve this, :mod:`unittest` supports some important concepts in an +To achieve this, :mod:`!unittest` supports some important concepts in an object-oriented way: test fixture @@ -33,7 +28,7 @@ test fixture test case A :dfn:`test case` is the individual unit of testing. It checks for a specific - response to a particular set of inputs. :mod:`unittest` provides a base class, + response to a particular set of inputs. :mod:`!unittest` provides a base class, :class:`TestCase`, which may be used to create new test cases. test suite @@ -53,7 +48,7 @@ test runner `Simple Smalltalk Testing: With Patterns `_ Kent Beck's original paper on testing frameworks using the pattern shared - by :mod:`unittest`. + by :mod:`!unittest`. `pytest `_ Third-party unittest framework with a lighter-weight syntax for writing @@ -81,7 +76,7 @@ test runner Basic example ------------- -The :mod:`unittest` module provides a rich set of tools for constructing and +The :mod:`!unittest` module provides a rich set of tools for constructing and running tests. This section demonstrates that a small subset of the tools suffice to meet the needs of most users. @@ -147,7 +142,7 @@ to enable a higher level of verbosity, and produce the following output:: OK -The above examples show the most commonly used :mod:`unittest` features which +The above examples show the most commonly used :mod:`!unittest` features which are sufficient to meet many everyday testing needs. The remainder of the documentation explores the full feature set from first principles. @@ -365,7 +360,7 @@ Organizing test code -------------------- The basic building blocks of unit testing are :dfn:`test cases` --- single -scenarios that must be set up and checked for correctness. In :mod:`unittest`, +scenarios that must be set up and checked for correctness. In :mod:`!unittest`, test cases are represented by :class:`unittest.TestCase` instances. To make your own test cases you must write subclasses of :class:`TestCase` or use :class:`FunctionTestCase`. @@ -387,7 +382,7 @@ testing code:: Note that in order to test something, we use one of the :ref:`assert\* methods ` provided by the :class:`TestCase` base class. If the test fails, an -exception will be raised with an explanatory message, and :mod:`unittest` +exception will be raised with an explanatory message, and :mod:`!unittest` will identify the test case as a :dfn:`failure`. Any other exceptions will be treated as :dfn:`errors`. @@ -438,12 +433,12 @@ run whether the test method succeeded or not. Such a working environment for the testing code is called a :dfn:`test fixture`. A new TestCase instance is created as a unique test fixture used to execute each individual test method. Thus -:meth:`~TestCase.setUp`, :meth:`~TestCase.tearDown`, and :meth:`~TestCase.__init__` +:meth:`~TestCase.setUp`, :meth:`~TestCase.tearDown`, and :meth:`!TestCase.__init__` will be called once per test. It is recommended that you use TestCase implementations to group tests together -according to the features they test. :mod:`unittest` provides a mechanism for -this: the :dfn:`test suite`, represented by :mod:`unittest`'s +according to the features they test. :mod:`!unittest` provides a mechanism for +this: the :dfn:`test suite`, represented by :mod:`!unittest`'s :class:`TestSuite` class. In most cases, calling :func:`unittest.main` will do the right thing and collect all the module's test cases for you and execute them. @@ -489,10 +484,10 @@ Re-using old test code ---------------------- Some users will find that they have existing test code that they would like to -run from :mod:`unittest`, without converting every old test function to a +run from :mod:`!unittest`, without converting every old test function to a :class:`TestCase` subclass. -For this reason, :mod:`unittest` provides a :class:`FunctionTestCase` class. +For this reason, :mod:`!unittest` provides a :class:`FunctionTestCase` class. This subclass of :class:`TestCase` can be used to wrap an existing test function. Set-up and tear-down functions can also be provided. @@ -513,12 +508,12 @@ set-up and tear-down methods:: .. note:: Even though :class:`FunctionTestCase` can be used to quickly convert an - existing test base over to a :mod:`unittest`\ -based system, this approach is + existing test base over to a :mod:`!unittest`\ -based system, this approach is not recommended. Taking the time to set up proper :class:`TestCase` subclasses will make future test refactorings infinitely easier. In some cases, the existing tests may have been written using the :mod:`doctest` -module. If so, :mod:`doctest` provides a :class:`DocTestSuite` class that can +module. If so, :mod:`doctest` provides a :class:`~doctest.DocTestSuite` class that can automatically build :class:`unittest.TestSuite` instances from the existing :mod:`doctest`\ -based tests. @@ -709,7 +704,7 @@ wouldn't be displayed:: Classes and functions --------------------- -This section describes in depth the API of :mod:`unittest`. +This section describes in depth the API of :mod:`!unittest`. .. _testcase-objects: @@ -720,7 +715,7 @@ Test cases .. class:: TestCase(methodName='runTest') Instances of the :class:`TestCase` class represent the logical test units - in the :mod:`unittest` universe. This class is intended to be used as a base + in the :mod:`!unittest` universe. This class is intended to be used as a base class, with specific tests being implemented by concrete subclasses. This class implements the interface needed by the test runner to allow it to drive the tests, and methods that the test code can use to check for and report various @@ -1023,7 +1018,7 @@ Test cases additional keyword argument *msg*. The context manager will store the caught exception object in its - :attr:`exception` attribute. This can be useful if the intention + :attr:`!exception` attribute. This can be useful if the intention is to perform additional checks on the exception raised:: with self.assertRaises(SomeException) as cm: @@ -1036,7 +1031,7 @@ Test cases Added the ability to use :meth:`assertRaises` as a context manager. .. versionchanged:: 3.2 - Added the :attr:`exception` attribute. + Added the :attr:`!exception` attribute. .. versionchanged:: 3.3 Added the *msg* keyword argument when used as a context manager. @@ -1089,8 +1084,8 @@ Test cases additional keyword argument *msg*. The context manager will store the caught warning object in its - :attr:`warning` attribute, and the source line which triggered the - warnings in the :attr:`filename` and :attr:`lineno` attributes. + :attr:`!warning` attribute, and the source line which triggered the + warnings in the :attr:`!filename` and :attr:`!lineno` attributes. This can be useful if the intention is to perform additional checks on the warning caught:: @@ -1177,7 +1172,7 @@ Test cases .. versionadded:: 3.4 - .. versionchanged:: next + .. versionchanged:: 3.15 Now accepts a *formatter* to control how messages are formatted. .. method:: assertNoLogs(logger=None, level=None) @@ -1228,9 +1223,9 @@ Test cases | :meth:`assertNotRegex(s, r) | ``not r.search(s)`` | 3.2 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertCountEqual(a, b) | *a* and *b* have the same | 3.2 | - | ` | elements in the same number, | | - | | regardless of their order. | | + | :meth:`assertCountEqual(a, b) | *a* contains the same elements | 3.2 | + | ` | as *b*, regardless of their | | + | | order. | | +---------------------------------------+--------------------------------+--------------+ | :meth:`assertStartsWith(a, b) | ``a.startswith(b)`` | 3.14 | | ` | | | @@ -1437,7 +1432,7 @@ Test cases that lists the differences between the sets. This method is used by default when comparing sets or frozensets with :meth:`assertEqual`. - Fails if either of *first* or *second* does not have a :meth:`set.difference` + Fails if either of *first* or *second* does not have a :meth:`~frozenset.difference` method. .. versionadded:: 3.1 @@ -1645,7 +1640,7 @@ Test cases .. method:: asyncSetUp() :async: - Method called to prepare the test fixture. This is called after :meth:`setUp`. + Method called to prepare the test fixture. This is called after :meth:`TestCase.setUp`. This is called immediately before calling the test method; other than :exc:`AssertionError` or :exc:`SkipTest`, any exception raised by this method will be considered an error rather than a test failure. The default implementation @@ -1655,7 +1650,7 @@ Test cases :async: Method called immediately after the test method has been called and the - result recorded. This is called before :meth:`tearDown`. This is called even if + result recorded. This is called before :meth:`~TestCase.tearDown`. This is called even if the test method raised an exception, so the implementation in subclasses may need to be particularly careful about checking internal state. Any exception, other than :exc:`AssertionError` or :exc:`SkipTest`, raised by this method will be @@ -1684,7 +1679,7 @@ Test cases Sets up a new event loop to run the test, collecting the result into the :class:`TestResult` object passed as *result*. If *result* is omitted or ``None``, a temporary result object is created (by calling - the :meth:`defaultTestResult` method) and used. The result object is + the :meth:`~TestCase.defaultTestResult` method) and used. The result object is returned to :meth:`run`'s caller. At the end of the test all the tasks in the event loop are cancelled. @@ -1734,7 +1729,7 @@ Test cases allows the test runner to drive the test, but does not provide the methods which test code can use to check and report errors. This is used to create test cases using legacy test code, allowing it to be integrated into a - :mod:`unittest`-based test framework. + :mod:`!unittest`-based test framework. .. _testsuite-objects: @@ -1805,7 +1800,7 @@ Grouping tests returned by repeated iterations before :meth:`TestSuite.run` must be the same for each call iteration. After :meth:`TestSuite.run`, callers should not rely on the tests returned by this method unless the caller uses a - subclass that overrides :meth:`TestSuite._removeTestAtIndex` to preserve + subclass that overrides :meth:`!TestSuite._removeTestAtIndex` to preserve test references. .. versionchanged:: 3.2 @@ -1816,10 +1811,10 @@ Grouping tests .. versionchanged:: 3.4 In earlier versions the :class:`TestSuite` held references to each :class:`TestCase` after :meth:`TestSuite.run`. Subclasses can restore - that behavior by overriding :meth:`TestSuite._removeTestAtIndex`. + that behavior by overriding :meth:`!TestSuite._removeTestAtIndex`. In the typical usage of a :class:`TestSuite` object, the :meth:`run` method - is invoked by a :class:`TestRunner` rather than by the end-user test harness. + is invoked by a :class:`!TestRunner` rather than by the end-user test harness. Loading and running tests @@ -1829,7 +1824,7 @@ Loading and running tests The :class:`TestLoader` class is used to create test suites from classes and modules. Normally, there is no need to create an instance of this class; the - :mod:`unittest` module provides an instance that can be shared as + :mod:`!unittest` module provides an instance that can be shared as :data:`unittest.defaultTestLoader`. Using a subclass or instance, however, allows customization of some configurable properties. @@ -1853,12 +1848,12 @@ Loading and running tests .. method:: loadTestsFromTestCase(testCaseClass) Return a suite of all test cases contained in the :class:`TestCase`\ -derived - :class:`testCaseClass`. + :class:`!testCaseClass`. A test case instance is created for each method named by :meth:`getTestCaseNames`. By default these are the method names beginning with ``test``. If :meth:`getTestCaseNames` returns no - methods, but the :meth:`runTest` method is implemented, a single test + methods, but the :meth:`!runTest` method is implemented, a single test case is created for that method instead. @@ -1905,13 +1900,13 @@ Loading and running tests case class will be picked up as "a test method within a test case class", rather than "a callable object". - For example, if you have a module :mod:`SampleTests` containing a - :class:`TestCase`\ -derived class :class:`SampleTestCase` with three test - methods (:meth:`test_one`, :meth:`test_two`, and :meth:`test_three`), the + For example, if you have a module :mod:`!SampleTests` containing a + :class:`TestCase`\ -derived class :class:`!SampleTestCase` with three test + methods (:meth:`!test_one`, :meth:`!test_two`, and :meth:`!test_three`), the specifier ``'SampleTests.SampleTestCase'`` would cause this method to return a suite which will run all three test methods. Using the specifier ``'SampleTests.SampleTestCase.test_two'`` would cause it to return a test - suite which will run only the :meth:`test_two` test method. The specifier + suite which will run only the :meth:`!test_two` test method. The specifier can refer to modules and packages which have not been imported; they will be imported as a side-effect. @@ -2055,10 +2050,10 @@ Loading and running tests properly recorded; test authors do not need to worry about recording the outcome of tests. - Testing frameworks built on top of :mod:`unittest` may want access to the + Testing frameworks built on top of :mod:`!unittest` may want access to the :class:`TestResult` object generated by running a set of tests for reporting purposes; a :class:`TestResult` instance is returned by the - :meth:`TestRunner.run` method for this purpose. + :meth:`!TestRunner.run` method for this purpose. :class:`TestResult` instances have the following attributes that will be of interest when inspecting the results of running a set of tests: @@ -2144,12 +2139,12 @@ Loading and running tests This method can be called to signal that the set of tests being run should be aborted by setting the :attr:`shouldStop` attribute to ``True``. - :class:`TestRunner` objects should respect this flag and return without + :class:`!TestRunner` objects should respect this flag and return without running any additional tests. For example, this feature is used by the :class:`TextTestRunner` class to stop the test framework when the user signals an interrupt from the - keyboard. Interactive tools which provide :class:`TestRunner` + keyboard. Interactive tools which provide :class:`!TestRunner` implementations can use this in a similar manner. The following methods of the :class:`TestResult` class are used to maintain @@ -2469,9 +2464,9 @@ Class and Module Fixtures ------------------------- Class and module level fixtures are implemented in :class:`TestSuite`. When -the test suite encounters a test from a new class then :meth:`tearDownClass` -from the previous class (if there is one) is called, followed by -:meth:`setUpClass` from the new class. +the test suite encounters a test from a new class then +:meth:`~TestCase.tearDownClass` from the previous class (if there is one) +is called, followed by :meth:`~TestCase.setUpClass` from the new class. Similarly if a test is from a different module from the previous test then ``tearDownModule`` from the previous module is run, followed by diff --git a/Doc/library/urllib.error.rst b/Doc/library/urllib.error.rst index 1686ddd09ca..b8864e36981 100644 --- a/Doc/library/urllib.error.rst +++ b/Doc/library/urllib.error.rst @@ -4,17 +4,14 @@ .. module:: urllib.error :synopsis: Exception classes raised by urllib.request. -.. moduleauthor:: Jeremy Hylton -.. sectionauthor:: Senthil Kumaran - **Source code:** :source:`Lib/urllib/error.py` -------------- -The :mod:`urllib.error` module defines the exception classes for exceptions +The :mod:`!urllib.error` module defines the exception classes for exceptions raised by :mod:`urllib.request`. The base exception class is :exc:`URLError`. -The following exceptions are raised by :mod:`urllib.error` as appropriate: +The following exceptions are raised by :mod:`!urllib.error` as appropriate: .. exception:: URLError diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 44a9c79cba2..ef48addaba0 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -35,7 +35,7 @@ Resource Locators. It supports the following URL schemes: ``file``, ``ftp``, macOS, it *may* be removed if CPython has been built with the :option:`--with-app-store-compliance` option. -The :mod:`urllib.parse` module defines functions that fall into two broad +The :mod:`!urllib.parse` module defines functions that fall into two broad categories: URL parsing and URL quoting. These are covered in detail in the following sections. @@ -50,12 +50,17 @@ URL Parsing The URL parsing functions focus on splitting a URL string into its components, or on combining URL components into a URL string. -.. function:: urlparse(urlstring, scheme='', allow_fragments=True) +.. function:: urlsplit(urlstring, scheme=None, allow_fragments=True, *, missing_as_none=False) - Parse a URL into six components, returning a 6-item :term:`named tuple`. This - corresponds to the general structure of a URL: - ``scheme://netloc/path;parameters?query#fragment``. - Each tuple item is a string, possibly empty. The components are not broken up + Parse a URL into five components, returning a 5-item :term:`named tuple` + :class:`SplitResult` or :class:`SplitResultBytes`. + This corresponds to the general structure of a URL: + ``scheme://netloc/path?query#fragment``. + Each tuple item is a string, possibly empty, or ``None`` if + *missing_as_none* is true. + Not defined component are represented an empty string (by default) or + ``None`` if *missing_as_none* is true. + The components are not broken up into smaller parts (for example, the network location is a single string), and % escapes are not expanded. The delimiters as shown above are not part of the result, except for a leading slash in the *path* component, which is retained if @@ -64,15 +69,15 @@ or on combining URL components into a URL string. .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> from urllib.parse import urlparse - >>> urlparse("scheme://netloc/path;parameters?query#fragment") - ParseResult(scheme='scheme', netloc='netloc', path='/path;parameters', params='', + >>> from urllib.parse import urlsplit + >>> urlsplit("scheme://netloc/path?query#fragment") + SplitResult(scheme='scheme', netloc='netloc', path='/path', query='query', fragment='fragment') - >>> o = urlparse("http://docs.python.org:80/3/library/urllib.parse.html?" + >>> o = urlsplit("http://docs.python.org:80/3/library/urllib.parse.html?" ... "highlight=params#url-parsing") >>> o - ParseResult(scheme='http', netloc='docs.python.org:80', - path='/3/library/urllib.parse.html', params='', + SplitResult(scheme='http', netloc='docs.python.org:80', + path='/3/library/urllib.parse.html', query='highlight=params', fragment='url-parsing') >>> o.scheme 'http' @@ -84,8 +89,14 @@ or on combining URL components into a URL string. 80 >>> o._replace(fragment="").geturl() 'http://docs.python.org:80/3/library/urllib.parse.html?highlight=params' + >>> urlsplit("http://docs.python.org?") + SplitResult(scheme='http', netloc='docs.python.org', path='', + query='', fragment='') + >>> urlsplit("http://docs.python.org?", missing_as_none=True) + SplitResult(scheme='http', netloc='docs.python.org', path='', + query='', fragment=None) - Following the syntax specifications in :rfc:`1808`, urlparse recognizes + Following the syntax specifications in :rfc:`1808`, :func:`!urlsplit` recognizes a netloc only if it is properly introduced by '//'. Otherwise the input is presumed to be a relative URL and thus to start with a path component. @@ -93,55 +104,58 @@ or on combining URL components into a URL string. .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> from urllib.parse import urlparse - >>> urlparse('//www.cwi.nl:80/%7Eguido/Python.html') - ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') - >>> urlparse('www.cwi.nl/%7Eguido/Python.html') - ParseResult(scheme='', netloc='', path='www.cwi.nl/%7Eguido/Python.html', - params='', query='', fragment='') - >>> urlparse('help/Python.html') - ParseResult(scheme='', netloc='', path='help/Python.html', params='', + >>> from urllib.parse import urlsplit + >>> urlsplit('//www.cwi.nl:80/%7Eguido/Python.html') + SplitResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', query='', fragment='') + >>> urlsplit('www.cwi.nl/%7Eguido/Python.html') + SplitResult(scheme='', netloc='', path='www.cwi.nl/%7Eguido/Python.html', + query='', fragment='') + >>> urlsplit('help/Python.html') + SplitResult(scheme='', netloc='', path='help/Python.html', + query='', fragment='') + >>> urlsplit('help/Python.html', missing_as_none=True) + SplitResult(scheme=None, netloc=None, path='help/Python.html', + query=None, fragment=None) The *scheme* argument gives the default addressing scheme, to be used only if the URL does not specify one. It should be the same type - (text or bytes) as *urlstring*, except that the default value ``''`` is + (text or bytes) as *urlstring* or ``None``, except that the ``''`` is always allowed, and is automatically converted to ``b''`` if appropriate. If the *allow_fragments* argument is false, fragment identifiers are not - recognized. Instead, they are parsed as part of the path, parameters - or query component, and :attr:`fragment` is set to the empty string in - the return value. + recognized. Instead, they are parsed as part of the path + or query component, and :attr:`fragment` is set to ``None`` or the empty + string (depending on the value of *missing_as_none*) in the return value. The return value is a :term:`named tuple`, which means that its items can be accessed by index or as named attributes, which are: - +------------------+-------+-------------------------+------------------------+ - | Attribute | Index | Value | Value if not present | - +==================+=======+=========================+========================+ - | :attr:`scheme` | 0 | URL scheme specifier | *scheme* parameter | - +------------------+-------+-------------------------+------------------------+ - | :attr:`netloc` | 1 | Network location part | empty string | - +------------------+-------+-------------------------+------------------------+ - | :attr:`path` | 2 | Hierarchical path | empty string | - +------------------+-------+-------------------------+------------------------+ - | :attr:`params` | 3 | Parameters for last | empty string | - | | | path element | | - +------------------+-------+-------------------------+------------------------+ - | :attr:`query` | 4 | Query component | empty string | - +------------------+-------+-------------------------+------------------------+ - | :attr:`fragment` | 5 | Fragment identifier | empty string | - +------------------+-------+-------------------------+------------------------+ - | :attr:`username` | | User name | :const:`None` | - +------------------+-------+-------------------------+------------------------+ - | :attr:`password` | | Password | :const:`None` | - +------------------+-------+-------------------------+------------------------+ - | :attr:`hostname` | | Host name (lower case) | :const:`None` | - +------------------+-------+-------------------------+------------------------+ - | :attr:`port` | | Port number as integer, | :const:`None` | - | | | if present | | - +------------------+-------+-------------------------+------------------------+ + +------------------+-------+-------------------------+-------------------------------+ + | Attribute | Index | Value | Value if not present | + +==================+=======+=========================+===============================+ + | :attr:`scheme` | 0 | URL scheme specifier | *scheme* parameter or | + | | | | empty string [1]_ | + +------------------+-------+-------------------------+-------------------------------+ + | :attr:`netloc` | 1 | Network location part | ``None`` or empty string [1]_ | + +------------------+-------+-------------------------+-------------------------------+ + | :attr:`path` | 2 | Hierarchical path | empty string | + +------------------+-------+-------------------------+-------------------------------+ + | :attr:`query` | 3 | Query component | ``None`` or empty string [1]_ | + +------------------+-------+-------------------------+-------------------------------+ + | :attr:`fragment` | 4 | Fragment identifier | ``None`` or empty string [1]_ | + +------------------+-------+-------------------------+-------------------------------+ + | :attr:`username` | | User name | ``None`` | + +------------------+-------+-------------------------+-------------------------------+ + | :attr:`password` | | Password | ``None`` | + +------------------+-------+-------------------------+-------------------------------+ + | :attr:`hostname` | | Host name (lower case) | ``None`` | + +------------------+-------+-------------------------+-------------------------------+ + | :attr:`port` | | Port number as integer, | ``None`` | + | | | if present | | + +------------------+-------+-------------------------+-------------------------------+ + + .. [1] Depending on the value of the *missing_as_none* argument. Reading the :attr:`port` attribute will raise a :exc:`ValueError` if an invalid port is specified in the URL. See section @@ -155,26 +169,30 @@ or on combining URL components into a URL string. ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is decomposed before parsing, no error will be raised. + Following some of the `WHATWG spec`_ that updates :rfc:`3986`, leading C0 + control and space characters are stripped from the URL. ``\n``, + ``\r`` and tab ``\t`` characters are removed from the URL at any position. + As is the case with all named tuples, the subclass has a few additional methods and attributes that are particularly useful. One such method is :meth:`_replace`. - The :meth:`_replace` method will return a new ParseResult object replacing specified - fields with new values. + The :meth:`_replace` method will return a new :class:`SplitResult` object + replacing specified fields with new values. .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> from urllib.parse import urlparse - >>> u = urlparse('//www.cwi.nl:80/%7Eguido/Python.html') + >>> from urllib.parse import urlsplit + >>> u = urlsplit('//www.cwi.nl:80/%7Eguido/Python.html') >>> u - ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') + SplitResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', + query='', fragment='') >>> u._replace(scheme='http') - ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') + SplitResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', + query='', fragment='') .. warning:: - :func:`urlparse` does not perform validation. See :ref:`URL parsing + :func:`urlsplit` does not perform validation. See :ref:`URL parsing security ` for details. .. versionchanged:: 3.2 @@ -187,12 +205,23 @@ or on combining URL components into a URL string. .. versionchanged:: 3.6 Out-of-range port numbers now raise :exc:`ValueError`, instead of - returning :const:`None`. + returning ``None``. .. versionchanged:: 3.8 Characters that affect netloc parsing under NFKC normalization will now raise :exc:`ValueError`. + .. versionchanged:: 3.10 + ASCII newline and tab characters are stripped from the URL. + + .. versionchanged:: 3.12 + Leading WHATWG C0 control and space characters are stripped from the URL. + + .. versionchanged:: 3.15 + Added the *missing_as_none* parameter. + +.. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser + .. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator='&') @@ -287,96 +316,63 @@ or on combining URL components into a URL string. separator key, with ``&`` as the default separator. -.. function:: urlunparse(parts) - - Construct a URL from a tuple as returned by ``urlparse()``. The *parts* - argument can be any six-item iterable. This may result in a slightly - different, but equivalent URL, if the URL that was parsed originally had - unnecessary delimiters (for example, a ``?`` with an empty query; the RFC - states that these are equivalent). - - -.. function:: urlsplit(urlstring, scheme='', allow_fragments=True) - - This is similar to :func:`urlparse`, but does not split the params from the URL. - This should generally be used instead of :func:`urlparse` if the more recent URL - syntax allowing parameters to be applied to each segment of the *path* portion - of the URL (see :rfc:`2396`) is wanted. A separate function is needed to - separate the path segments and parameters. This function returns a 5-item - :term:`named tuple`:: - - (addressing scheme, network location, path, query, fragment identifier). - - The return value is a :term:`named tuple`, its items can be accessed by index - or as named attributes: - - +------------------+-------+-------------------------+----------------------+ - | Attribute | Index | Value | Value if not present | - +==================+=======+=========================+======================+ - | :attr:`scheme` | 0 | URL scheme specifier | *scheme* parameter | - +------------------+-------+-------------------------+----------------------+ - | :attr:`netloc` | 1 | Network location part | empty string | - +------------------+-------+-------------------------+----------------------+ - | :attr:`path` | 2 | Hierarchical path | empty string | - +------------------+-------+-------------------------+----------------------+ - | :attr:`query` | 3 | Query component | empty string | - +------------------+-------+-------------------------+----------------------+ - | :attr:`fragment` | 4 | Fragment identifier | empty string | - +------------------+-------+-------------------------+----------------------+ - | :attr:`username` | | User name | :const:`None` | - +------------------+-------+-------------------------+----------------------+ - | :attr:`password` | | Password | :const:`None` | - +------------------+-------+-------------------------+----------------------+ - | :attr:`hostname` | | Host name (lower case) | :const:`None` | - +------------------+-------+-------------------------+----------------------+ - | :attr:`port` | | Port number as integer, | :const:`None` | - | | | if present | | - +------------------+-------+-------------------------+----------------------+ - - Reading the :attr:`port` attribute will raise a :exc:`ValueError` if - an invalid port is specified in the URL. See section - :ref:`urlparse-result-object` for more information on the result object. - - Unmatched square brackets in the :attr:`netloc` attribute will raise a - :exc:`ValueError`. - - Characters in the :attr:`netloc` attribute that decompose under NFKC - normalization (as used by the IDNA encoding) into any of ``/``, ``?``, - ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is - decomposed before parsing, no error will be raised. - - Following some of the `WHATWG spec`_ that updates RFC 3986, leading C0 - control and space characters are stripped from the URL. ``\n``, - ``\r`` and tab ``\t`` characters are removed from the URL at any position. - - .. warning:: - - :func:`urlsplit` does not perform validation. See :ref:`URL parsing - security ` for details. - - .. versionchanged:: 3.6 - Out-of-range port numbers now raise :exc:`ValueError`, instead of - returning :const:`None`. - - .. versionchanged:: 3.8 - Characters that affect netloc parsing under NFKC normalization will - now raise :exc:`ValueError`. - - .. versionchanged:: 3.10 - ASCII newline and tab characters are stripped from the URL. - - .. versionchanged:: 3.12 - Leading WHATWG C0 control and space characters are stripped from the URL. - -.. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser - .. function:: urlunsplit(parts) + urlunsplit(parts, *, keep_empty) - Combine the elements of a tuple as returned by :func:`urlsplit` into a - complete URL as a string. The *parts* argument can be any five-item - iterable. This may result in a slightly different, but equivalent URL, if the - URL that was parsed originally had unnecessary delimiters (for example, a ? - with an empty query; the RFC states that these are equivalent). + Construct a URL from a tuple as returned by :func:`urlsplit`. The *parts* + argument can be any five-item iterable. + + This may result in a slightly different, but equivalent URL, if the + URL that was parsed originally had unnecessary delimiters (for example, + a ``?`` with an empty query; the RFC states that these are equivalent). + + If *keep_empty* is true, empty strings are kept in the result (for example, + a ``?`` for an empty query), only ``None`` components are omitted. + This allows rebuilding a URL that was parsed with option + ``missing_as_none=True``. + By default, *keep_empty* is true if *parts* is the result of the + :func:`urlsplit` call with ``missing_as_none=True``. + + .. versionchanged:: 3.15 + Added the *keep_empty* parameter. + + +.. function:: urlparse(urlstring, scheme=None, allow_fragments=True, *, missing_as_none=False) + + This is similar to :func:`urlsplit`, but additionally splits the *path* + component on *path* and *params*. + This function returns a 6-item :term:`named tuple` :class:`ParseResult` + or :class:`ParseResultBytes`. + Its items are the same as for the :func:`!urlsplit` result, except that + *params* is inserted at index 3, between *path* and *query*. + + This function is based on obsoleted :rfc:`1738` and :rfc:`1808`, which + listed *params* as the main URL component. + The more recent URL syntax allows parameters to be applied to each segment + of the *path* portion of the URL (see :rfc:`3986`). + :func:`urlsplit` should generally be used instead of :func:`urlparse`. + A separate function is needed to separate the path segments and parameters. + +.. function:: urlunparse(parts) + urlunparse(parts, *, keep_empty) + + Combine the elements of a tuple as returned by :func:`urlparse` into a + complete URL as a string. The *parts* argument can be any six-item + iterable. + + This may result in a slightly different, but equivalent URL, if the + URL that was parsed originally had unnecessary delimiters (for example, + a ``?`` with an empty query; the RFC states that these are equivalent). + + If *keep_empty* is true, empty strings are kept in the result (for example, + a ``?`` for an empty query), only ``None`` components are omitted. + This allows rebuilding a URL that was parsed with option + ``missing_as_none=True``. + By default, *keep_empty* is true if *parts* is the result of the + :func:`urlparse` call with ``missing_as_none=True``. + + .. versionchanged:: 3.15 + Added the *keep_empty* parameter. .. function:: urljoin(base, url, allow_fragments=True) @@ -391,7 +387,7 @@ or on combining URL components into a URL string. 'http://www.cwi.nl/%7Eguido/FAQ.html' The *allow_fragments* argument has the same meaning and default as for - :func:`urlparse`. + :func:`urlsplit`. .. note:: @@ -422,23 +418,25 @@ or on combining URL components into a URL string. Behavior updated to match the semantics defined in :rfc:`3986`. -.. function:: urldefrag(url) +.. function:: urldefrag(url, *, missing_as_none=False) If *url* contains a fragment identifier, return a modified version of *url* with no fragment identifier, and the fragment identifier as a separate string. If there is no fragment identifier in *url*, return *url* unmodified - and an empty string. + and an empty string (by default) or ``None`` if *missing_as_none* is true. The return value is a :term:`named tuple`, its items can be accessed by index or as named attributes: - +------------------+-------+-------------------------+----------------------+ - | Attribute | Index | Value | Value if not present | - +==================+=======+=========================+======================+ - | :attr:`url` | 0 | URL with no fragment | empty string | - +------------------+-------+-------------------------+----------------------+ - | :attr:`fragment` | 1 | Fragment identifier | empty string | - +------------------+-------+-------------------------+----------------------+ + +------------------+-------+-------------------------+-------------------------------+ + | Attribute | Index | Value | Value if not present | + +==================+=======+=========================+===============================+ + | :attr:`url` | 0 | URL with no fragment | empty string | + +------------------+-------+-------------------------+-------------------------------+ + | :attr:`fragment` | 1 | Fragment identifier | ``None`` or empty string [3]_ | + +------------------+-------+-------------------------+-------------------------------+ + + .. [3] Depending on the value of the *missing_as_none* argument. See section :ref:`urlparse-result-object` for more information on the result object. @@ -446,6 +444,9 @@ or on combining URL components into a URL string. .. versionchanged:: 3.2 Result is a structured object rather than a simple 2-tuple. + .. versionchanged:: 3.15 + Added the *missing_as_none* parameter. + .. function:: unwrap(url) Extract the url from a wrapped URL (that is, a string formatted as @@ -465,8 +466,9 @@ URLs elsewhere. Their purpose is for practical functionality rather than purity. Instead of raising an exception on unusual input, they may instead return some -component parts as empty strings. Or components may contain more than perhaps -they should. +component parts as empty strings or ``None`` (depending on the value of the +*missing_as_none* argument). +Or components may contain more than perhaps they should. We recommend that users of these APIs where the values may be used anywhere with security implications code defensively. Do some verification within your @@ -531,7 +533,7 @@ individual URL quoting functions. Structured Parse Results ------------------------ -The result objects from the :func:`urlparse`, :func:`urlsplit` and +The result objects from the :func:`urlsplit`, :func:`urlparse` and :func:`urldefrag` functions are subclasses of the :class:`tuple` type. These subclasses add the attributes listed in the documentation for those functions, the encoding and decoding support described in the @@ -542,7 +544,8 @@ previous section, as well as an additional method: Return the re-combined version of the original URL as a string. This may differ from the original URL in that the scheme may be normalized to lower case and empty components may be dropped. Specifically, empty parameters, - queries, and fragment identifiers will be removed. + queries, and fragment identifiers will be removed unless the URL was parsed + with ``missing_as_none=True``. For :func:`urldefrag` results, only empty fragment identifiers will be removed. For :func:`urlsplit` and :func:`urlparse` results, all noted changes will be @@ -559,6 +562,9 @@ previous section, as well as an additional method: >>> r2 = urlsplit(r1.geturl()) >>> r2.geturl() 'http://www.Python.org/doc/' + >>> r3 = urlsplit(url, missing_as_none=True) + >>> r3.geturl() + 'http://www.Python.org/doc/#' The following classes provide the implementations of the structured parse diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 5f796578eaa..64e915d042d 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -4,15 +4,11 @@ .. module:: urllib.request :synopsis: Extensible library for opening URLs. -.. moduleauthor:: Jeremy Hylton -.. sectionauthor:: Moshe Zadka -.. sectionauthor:: Senthil Kumaran - **Source code:** :source:`Lib/urllib/request.py` -------------- -The :mod:`urllib.request` module defines functions and classes which help in +The :mod:`!urllib.request` module defines functions and classes which help in opening URLs (mostly HTTP) in a complex world --- basic and digest authentication, redirections, cookies and more. @@ -31,7 +27,7 @@ authentication, redirections, cookies and more. .. include:: ../includes/wasm-notavail.rst -The :mod:`urllib.request` module defines the following functions: +The :mod:`!urllib.request` module defines the following functions: .. function:: urlopen(url, data=None[, timeout], *, context=None) @@ -1485,8 +1481,8 @@ some point in the future. calls to :func:`urlretrieve`. -:mod:`urllib.request` Restrictions ----------------------------------- +:mod:`!urllib.request` Restrictions +----------------------------------- .. index:: pair: HTTP; protocol @@ -1539,15 +1535,15 @@ some point in the future. -:mod:`urllib.response` --- Response classes used by urllib -========================================================== +:mod:`!urllib.response` --- Response classes used by urllib +=========================================================== .. module:: urllib.response :synopsis: Response classes used by urllib. -The :mod:`urllib.response` module defines functions and classes which define a +The :mod:`!urllib.response` module defines functions and classes which define a minimal file-like interface, including ``read()`` and ``readline()``. -Functions defined by this module are used internally by the :mod:`urllib.request` module. +Functions defined by this module are used internally by the :mod:`!urllib.request` module. The typical response object is a :class:`urllib.response.addinfourl` instance: .. class:: addinfourl diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index 016fcdc75da..492c65ae209 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -5,8 +5,6 @@ :synopsis: Load a robots.txt file and answer questions about fetchability of other URLs. -.. sectionauthor:: Skip Montanaro - **Source code:** :source:`Lib/urllib/robotparser.py` .. index:: @@ -19,7 +17,7 @@ This module provides a single class, :class:`RobotFileParser`, which answers questions about whether or not a particular user agent can fetch a URL on the -web site that published the :file:`robots.txt` file. For more details on the +website that published the :file:`robots.txt` file. For more details on the structure of :file:`robots.txt` files, see http://www.robotstxt.org/orig.html. @@ -91,16 +89,16 @@ class:: >>> import urllib.robotparser >>> rp = urllib.robotparser.RobotFileParser() - >>> rp.set_url("http://www.musi-cal.com/robots.txt") + >>> rp.set_url("http://www.pythontest.net/robots.txt") >>> rp.read() >>> rrate = rp.request_rate("*") >>> rrate.requests - 3 + 1 >>> rrate.seconds - 20 + 1 >>> rp.crawl_delay("*") 6 - >>> rp.can_fetch("*", "http://www.musi-cal.com/cgi-bin/search?city=San+Francisco") - False - >>> rp.can_fetch("*", "http://www.musi-cal.com/") + >>> rp.can_fetch("*", "http://www.pythontest.net/") True + >>> rp.can_fetch("*", "http://www.pythontest.net/no-robots-here/") + False diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index aa4f1bf940b..4b505c81c06 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -3,8 +3,6 @@ .. module:: uuid :synopsis: UUID objects (universally unique identifiers) according to RFC 9562 -.. moduleauthor:: Ka-Ping Yee -.. sectionauthor:: George Yoshida **Source code:** :source:`Lib/uuid.py` @@ -168,7 +166,7 @@ which relays any information about the UUID's safety, using this enumeration: .. versionadded:: 3.7 -The :mod:`uuid` module defines the following functions: +The :mod:`!uuid` module defines the following functions: .. function:: getnode() @@ -273,7 +271,7 @@ The :mod:`uuid` module defines the following functions: .. versionadded:: 3.14 -The :mod:`uuid` module defines the following namespace identifiers for use with +The :mod:`!uuid` module defines the following namespace identifiers for use with :func:`uuid3` or :func:`uuid5`. @@ -298,7 +296,7 @@ The :mod:`uuid` module defines the following namespace identifiers for use with When this namespace is specified, the *name* string is an X.500 DN in DER or a text output format. -The :mod:`uuid` module defines the following constants for the possible values +The :mod:`!uuid` module defines the following constants for the possible values of the :attr:`~UUID.variant` attribute: @@ -324,7 +322,7 @@ of the :attr:`~UUID.variant` attribute: Reserved for future definition. -The :mod:`uuid` module defines the special Nil and Max UUID values: +The :mod:`!uuid` module defines the special Nil and Max UUID values: .. data:: NIL @@ -357,7 +355,7 @@ Command-Line Usage .. versionadded:: 3.12 -The :mod:`uuid` module can be executed as a script from the command line. +The :mod:`!uuid` module can be executed as a script from the command line. .. code-block:: sh @@ -406,7 +404,7 @@ The following options are accepted: Example ------- -Here are some examples of typical usage of the :mod:`uuid` module:: +Here are some examples of typical usage of the :mod:`!uuid` module:: >>> import uuid @@ -473,7 +471,7 @@ Here are some examples of typical usage of the :mod:`uuid` module:: Command-Line Example -------------------- -Here are some examples of typical usage of the :mod:`uuid` command-line interface: +Here are some examples of typical usage of the :mod:`!uuid` command-line interface: .. code-block:: shell diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index b0eb8ee18fa..8bb267d5a0b 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -4,9 +4,6 @@ .. module:: venv :synopsis: Creation of virtual environments. -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - .. versionadded:: 3.3 **Source code:** :source:`Lib/venv/` @@ -407,6 +404,8 @@ creation according to their needs, the :class:`EnvBuilder` class. * ``lib_path`` - The purelib path for the virtual environment. + * ``platlib_path`` - The platlib path for the virtual environment. + * ``bin_path`` - The script path for the virtual environment. * ``bin_name`` - The name of the script path relative to the virtual @@ -431,6 +430,9 @@ creation according to their needs, the :class:`EnvBuilder` class. The attribute ``lib_path`` was added to the context, and the context object was documented. + .. versionchanged:: 3.15 + The attribute ``platlib_path`` was added to the context. + .. method:: create_configuration(context) Creates the ``pyvenv.cfg`` configuration file in the environment. @@ -545,7 +547,7 @@ subclass which installs setuptools and pip into a created virtual environment:: from subprocess import Popen, PIPE import sys from threading import Thread - from urllib.parse import urlparse + from urllib.parse import urlsplit from urllib.request import urlretrieve import venv @@ -616,7 +618,7 @@ subclass which installs setuptools and pip into a created virtual environment:: stream.close() def install_script(self, context, name, url): - _, _, path, _, _, _ = urlparse(url) + _, _, path, _, _ = urlsplit(url) fn = os.path.split(path)[-1] binpath = context.bin_path distpath = os.path.join(binpath, fn) diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index b6392450a17..b25384dbfce 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -159,8 +159,10 @@ the disposition of the match. Each entry is a tuple of the form (*action*, * *message* is a string containing a regular expression that the start of the warning message must match, case-insensitively. In :option:`-W` and - :envvar:`PYTHONWARNINGS`, *message* is a literal string that the start of the - warning message must contain (case-insensitively), ignoring any whitespace at + :envvar:`PYTHONWARNINGS`, if *message* starts and ends with a forward slash + (``/``), it specifies a regular expression as above; + otherwise it is a literal string that the start of the + warning message must match (case-insensitively), ignoring any whitespace at the start or end of *message*. * *category* is a class (a subclass of :exc:`Warning`) of which the warning @@ -168,7 +170,9 @@ the disposition of the match. Each entry is a tuple of the form (*action*, * *module* is a string containing a regular expression that the start of the fully qualified module name must match, case-sensitively. In :option:`-W` and - :envvar:`PYTHONWARNINGS`, *module* is a literal string that the + :envvar:`PYTHONWARNINGS`, if *module* starts and ends with a forward slash + (``/``), it specifies a regular expression as above; + otherwise it is a literal string that the fully qualified module name must be equal to (case-sensitively), ignoring any whitespace at the start or end of *module*. @@ -203,7 +207,7 @@ Describing Warning Filters The warnings filter is initialized by :option:`-W` options passed to the Python interpreter command line and the :envvar:`PYTHONWARNINGS` environment variable. The interpreter saves the arguments for all supplied entries without -interpretation in :data:`sys.warnoptions`; the :mod:`warnings` module parses these +interpretation in :data:`sys.warnoptions`; the :mod:`!warnings` module parses these when it is first imported (invalid options are ignored, after printing a message to :data:`sys.stderr`). @@ -476,14 +480,28 @@ Available Functions .. function:: warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, source=None) This is a low-level interface to the functionality of :func:`warn`, passing in - explicitly the message, category, filename and line number, and optionally the - module name and the registry (which should be the ``__warningregistry__`` - dictionary of the module). The module name defaults to the filename with - ``.py`` stripped; if no registry is passed, the warning is never suppressed. + explicitly the message, category, filename and line number, and optionally + other arguments. *message* must be a string and *category* a subclass of :exc:`Warning` or *message* may be a :exc:`Warning` instance, in which case *category* will be ignored. + *module*, if supplied, should be the module name. + If no module is passed, the module regular expression in + :ref:`warnings filter ` will be tested against the module + names constructed from the path components starting from all parent + directories (with ``/__init__.py``, ``.py`` and, on Windows, ``.pyw`` + stripped) and against the filename with ``.py`` stripped. + For example, when the filename is ``'/path/to/package/module.py'``, it will + be tested against ``'path.to.package.module'``, ``'to.package.module'`` + ``'package.module'``, ``'module'``, and ``'/path/to/package/module'``. + + *registry*, if supplied, should be the ``__warningregistry__`` dictionary + of the module. + If no registry is passed, each warning is treated as the first occurrence, + that is, filter actions ``"default"``, ``"module"`` and ``"once"`` are + handled as ``"always"``. + *module_globals*, if supplied, should be the global namespace in use by the code for which the warning is issued. (This argument is used to support displaying source for modules found in zipfiles or other non-filesystem import @@ -495,6 +513,10 @@ Available Functions .. versionchanged:: 3.6 Add the *source* parameter. + .. versionchanged:: 3.15 + If no module is passed, test the filter regular expression against + module names created from the path, not only the path itself. + .. function:: showwarning(message, category, filename, lineno, file=None, line=None) @@ -586,7 +608,7 @@ Available Functions The deprecation message passed to the decorator is saved in the ``__deprecated__`` attribute on the decorated object. If applied to an overload, the decorator - must be after the :func:`@overload ` decorator + must be after the :deco:`~typing.overload` decorator for the attribute to exist on the overload as returned by :func:`typing.get_overloads`. @@ -609,8 +631,8 @@ Available Context Managers :func:`showwarning`. The *module* argument takes a module that will be used instead of the - module returned when you import :mod:`warnings` whose filter will be - protected. This argument exists primarily for testing the :mod:`warnings` + module returned when you import :mod:`!warnings` whose filter will be + protected. This argument exists primarily for testing the :mod:`!warnings` module itself. If the *action* argument is not ``None``, the remaining arguments are @@ -647,7 +669,7 @@ to true for free-threaded builds and false otherwise. If the :data:`~sys.flags.context_aware_warnings` flag is false, then :class:`catch_warnings` will modify the global attributes of the -:mod:`warnings` module. This is not safe if used within a concurrent program +:mod:`!warnings` module. This is not safe if used within a concurrent program (using multiple threads or using asyncio coroutines). For example, if two or more threads use the :class:`catch_warnings` class at the same time, the behavior is undefined. diff --git a/Doc/library/wave.rst b/Doc/library/wave.rst index a3f5bfd5e2f..d320975708c 100644 --- a/Doc/library/wave.rst +++ b/Doc/library/wave.rst @@ -4,29 +4,32 @@ .. module:: wave :synopsis: Provide an interface to the WAV sound format. -.. sectionauthor:: Moshe Zadka -.. Documentations stolen from comments in file. - **Source code:** :source:`Lib/wave.py` -------------- -The :mod:`wave` module provides a convenient interface to the Waveform Audio -"WAVE" (or "WAV") file format. Only uncompressed PCM encoded wave files are -supported. +The :mod:`!wave` module provides a convenient interface to the Waveform Audio +"WAVE" (or "WAV") file format. + +The module supports uncompressed PCM and IEEE floating-point WAV formats. .. versionchanged:: 3.12 Support for ``WAVE_FORMAT_EXTENSIBLE`` headers was added, provided that the extended format is ``KSDATAFORMAT_SUBTYPE_PCM``. -The :mod:`wave` module defines the following function and exception: +.. versionchanged:: 3.15 + + Support for reading and writing ``WAVE_FORMAT_IEEE_FLOAT`` files was added. + +The :mod:`!wave` module defines the following function and exception: .. function:: open(file, mode=None) - If *file* is a string, open the file by that name, otherwise treat it as a - file-like object. *mode* can be: + If *file* is a string, a :term:`path-like object` or a + :term:`bytes-like object` open the file by that name, otherwise treat it as + a file-like object. *mode* can be: ``'rb'`` Read only mode. @@ -52,12 +55,31 @@ The :mod:`wave` module defines the following function and exception: .. versionchanged:: 3.4 Added support for unseekable files. + .. versionchanged:: 3.15 + Added support for :term:`path-like objects ` + and :term:`bytes-like objects `. + .. exception:: Error An error raised when something is impossible because it violates the WAV specification or hits an implementation deficiency. +.. data:: WAVE_FORMAT_PCM + + Format code for uncompressed PCM audio. + + +.. data:: WAVE_FORMAT_IEEE_FLOAT + + Format code for IEEE floating-point audio. + + +.. data:: WAVE_FORMAT_EXTENSIBLE + + Format code for WAVE extensible headers. + + .. _wave-read-objects: Wave_read Objects @@ -72,7 +94,7 @@ Wave_read Objects .. method:: close() - Close the stream if it was opened by :mod:`wave`, and make the instance + Close the stream if it was opened by :mod:`!wave`, and make the instance unusable. This is called automatically on object collection. @@ -96,6 +118,14 @@ Wave_read Objects Returns number of audio frames. + .. method:: getformat() + + Returns the frame format code. + + This is one of :data:`WAVE_FORMAT_PCM`, + :data:`WAVE_FORMAT_IEEE_FLOAT`, or :data:`WAVE_FORMAT_EXTENSIBLE`. + + .. method:: getcomptype() Returns compression type (``'NONE'`` is the only supported type). @@ -110,8 +140,8 @@ Wave_read Objects .. method:: getparams() Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth, - framerate, nframes, comptype, compname)``, equivalent to output of the - ``get*()`` methods. + framerate, nframes, comptype, compname)``, equivalent to output + of the ``get*()`` methods. .. method:: readframes(n) @@ -169,7 +199,7 @@ Wave_write Objects .. method:: close() Make sure *nframes* is correct, and close the file if it was opened by - :mod:`wave`. This method is called upon object collection. It will raise + :mod:`!wave`. This method is called upon object collection. It will raise an exception if the output stream is not seekable and *nframes* does not match the number of frames actually written. @@ -179,10 +209,23 @@ Wave_write Objects Set the number of channels. + .. method:: getnchannels() + + Return the number of channels. + + .. method:: setsampwidth(n) Set the sample width to *n* bytes. + For :data:`WAVE_FORMAT_IEEE_FLOAT`, only 4-byte (32-bit) and + 8-byte (64-bit) sample widths are supported. + + + .. method:: getsampwidth() + + Return the sample width in bytes. + .. method:: setframerate(n) @@ -193,6 +236,11 @@ Wave_write Objects integer. + .. method:: getframerate() + + Return the frame rate. + + .. method:: setnframes(n) Set the number of frames to *n*. This will be changed later if the number @@ -200,17 +248,60 @@ Wave_write Objects raise an error if the output stream is not seekable). + .. method:: getnframes() + + Return the number of audio frames written so far. + + .. method:: setcomptype(type, name) Set the compression type and description. At the moment, only compression type ``NONE`` is supported, meaning no compression. + .. method:: getcomptype() + + Return the compression type (``'NONE'``). + + + .. method:: getcompname() + + Return the human-readable compression type name. + + + .. method:: setformat(format) + + Set the frame format code. + + Supported values are :data:`WAVE_FORMAT_PCM` and + :data:`WAVE_FORMAT_IEEE_FLOAT`. + + When setting :data:`WAVE_FORMAT_IEEE_FLOAT`, the sample width must be + 4 or 8 bytes. + + + .. method:: getformat() + + Return the current frame format code. + + .. method:: setparams(tuple) - The *tuple* should be ``(nchannels, sampwidth, framerate, nframes, comptype, - compname)``, with values valid for the ``set*()`` methods. Sets all - parameters. + The *tuple* should be + ``(nchannels, sampwidth, framerate, nframes, comptype, compname, format)``, + with values valid for the ``set*()`` methods. Sets all parameters. + + For backwards compatibility, a 6-item tuple without *format* is also + accepted and defaults to :data:`WAVE_FORMAT_PCM`. + + For ``format=WAVE_FORMAT_IEEE_FLOAT``, *sampwidth* must be 4 or 8. + + + .. method:: getparams() + + Return a :func:`~collections.namedtuple` + ``(nchannels, sampwidth, framerate, nframes, comptype, compname)`` + containing the current output parameters. .. method:: tell() @@ -240,3 +331,6 @@ Wave_write Objects Note that it is invalid to set any parameters after calling :meth:`writeframes` or :meth:`writeframesraw`, and any attempt to do so will raise :exc:`wave.Error`. + + For :data:`WAVE_FORMAT_IEEE_FLOAT` output, a ``fact`` chunk is written as + required by the WAVE specification for non-PCM formats. diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 2a25ed045c6..87dd860da4d 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -1,21 +1,16 @@ .. _mod-weakref: -:mod:`weakref` --- Weak references -================================== +:mod:`!weakref` --- Weak references +=================================== .. module:: weakref :synopsis: Support for weak references and weak dictionaries. -.. moduleauthor:: Fred L. Drake, Jr. -.. moduleauthor:: Neil Schemenauer -.. moduleauthor:: Martin von Löwis -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/weakref.py` -------------- -The :mod:`weakref` module allows the Python programmer to create :dfn:`weak +The :mod:`!weakref` module allows the Python programmer to create :dfn:`weak references` to objects. .. When making changes to the examples in this file, be sure to update @@ -39,7 +34,7 @@ associate a name with each. If you used a Python dictionary to map names to images, or images to names, the image objects would remain alive just because they appeared as values or keys in the dictionaries. The :class:`WeakKeyDictionary` and :class:`WeakValueDictionary` classes supplied by -the :mod:`weakref` module are an alternative, using weak references to construct +the :mod:`!weakref` module are an alternative, using weak references to construct mappings that don't keep objects alive solely because they appear in the mapping objects. If, for example, an image object is a value in a :class:`WeakValueDictionary`, then when the last remaining references to that @@ -63,7 +58,7 @@ remains alive until the object is collected. Most programs should find that using one of these weak container types or :class:`finalize` is all they need -- it's not usually necessary to create your own weak references directly. The low-level machinery is -exposed by the :mod:`weakref` module for the benefit of advanced uses. +exposed by the :mod:`!weakref` module for the benefit of advanced uses. Not all objects can be weakly referenced. Objects which support weak references include class instances, functions written in Python (but not in C), instance methods, diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index fd6abc70261..389648d4f39 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -4,14 +4,11 @@ .. module:: webbrowser :synopsis: Easy-to-use controller for web browsers. -.. moduleauthor:: Fred L. Drake, Jr. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/webbrowser.py` -------------- -The :mod:`webbrowser` module provides a high-level interface to allow displaying +The :mod:`!webbrowser` module provides a high-level interface to allow displaying web-based documents to users. Under most circumstances, simply calling the :func:`.open` function from this module will do the right thing. @@ -46,9 +43,14 @@ On iOS, the :envvar:`BROWSER` environment variable, as well as any arguments controlling autoraise, browser preference, and new tab/window creation will be ignored. Web pages will *always* be opened in the user's preferred browser, in a new tab, with the browser being brought to the foreground. The use of the -:mod:`webbrowser` module on iOS requires the :mod:`ctypes` module. If +:mod:`!webbrowser` module on iOS requires the :mod:`ctypes` module. If :mod:`ctypes` isn't available, calls to :func:`.open` will fail. +.. _webbrowser-cli: + +Command-line interface +---------------------- + .. program:: webbrowser The script :program:`webbrowser` can be used as a command-line interface for the @@ -232,7 +234,7 @@ Here are some simple examples:: .. _browser-controllers: -Browser Controller Objects +Browser controller objects -------------------------- Browser controllers provide the :attr:`~controller.name` attribute, diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst index 61056d41cf7..2353bfd5281 100644 --- a/Doc/library/winreg.rst +++ b/Doc/library/winreg.rst @@ -2,10 +2,9 @@ ========================================== .. module:: winreg - :platform: Windows :synopsis: Routines and objects for manipulating the Windows registry. -.. sectionauthor:: Mark Hammond +**Source code:** :source:`PC/winreg.c` -------------- @@ -14,6 +13,8 @@ integer as the registry handle, a :ref:`handle object ` is used to ensure that the handles are closed correctly, even if the programmer neglects to explicitly close them. +.. availability:: Windows. + .. _exception-changed: .. versionchanged:: 3.3 @@ -23,7 +24,7 @@ to explicitly close them. .. _functions: Functions ------------------- +--------- This module offers the following functions: @@ -188,7 +189,7 @@ This module offers the following functions: .. audit-event:: winreg.DeleteTree key,sub_key winreg.DeleteTree - .. versionadded:: next + .. versionadded:: 3.15 .. function:: DeleteValue(key, value) @@ -552,9 +553,9 @@ This module offers the following functions: .. _constants: Constants ------------------- +--------- -The following constants are defined for use in many :mod:`winreg` functions. +The following constants are defined for use in many :mod:`!winreg` functions. .. _hkey-constants: @@ -771,8 +772,9 @@ Handle objects provide semantics for :meth:`~object.__bool__` -- thus :: will print ``Yes`` if the handle is currently valid (has not been closed or detached). -The object also support comparison semantics, so handle objects will compare -true if they both reference the same underlying Windows handle value. +The object also support equality comparison semantics, so handle objects will +compare equal if they both reference the same underlying Windows handle value. +Closed handle objects (those with a handle value of zero) always compare equal. Handle objects can be converted to an integer (e.g., using the built-in :func:`int` function), in which case the underlying Windows handle value is @@ -815,3 +817,6 @@ integer handle, and also disconnect the Windows handle from the handle object. will automatically close *key* when control leaves the :keyword:`with` block. +.. versionchanged:: 3.15 + Handle objects are now compared by their underlying Windows handle value + instead of object identity for equality comparisons. diff --git a/Doc/library/winsound.rst b/Doc/library/winsound.rst index 925984c3cdb..6b5f8802148 100644 --- a/Doc/library/winsound.rst +++ b/Doc/library/winsound.rst @@ -2,17 +2,20 @@ ======================================================== .. module:: winsound - :platform: Windows :synopsis: Access to the sound-playing machinery for Windows. -.. moduleauthor:: Toby Dickenson -.. sectionauthor:: Fred L. Drake, Jr. +**Source code:** :source:`PC/winsound.c` -------------- -The :mod:`winsound` module provides access to the basic sound-playing machinery +The :mod:`!winsound` module provides access to the basic sound-playing machinery provided by Windows platforms. It includes functions and several constants. +.. availability:: Windows. + + +Functions +--------- .. function:: Beep(frequency, duration) @@ -44,6 +47,9 @@ provided by Windows platforms. It includes functions and several constants. error, :exc:`RuntimeError` is raised. +Constants +--------- + .. data:: SND_FILENAME The *sound* parameter is the name of a WAV file. Do not use with diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index 381c9938347..2af54dc2a7e 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -4,13 +4,15 @@ .. module:: wsgiref :synopsis: WSGI Utilities and Reference Implementation. -.. moduleauthor:: Phillip J. Eby -.. sectionauthor:: Phillip J. Eby - **Source code:** :source:`Lib/wsgiref` -------------- +.. warning:: + + :mod:`!wsgiref` is a reference implementation and is not recommended for + production. The module only implements basic security checks. + The Web Server Gateway Interface (WSGI) is a standard interface between web server software and web applications written in Python. Having a standard interface makes it easy to use an application that supports WSGI with a number @@ -21,7 +23,7 @@ and corner case of the WSGI design. You don't need to understand every detail of WSGI just to install a WSGI application or to write a web application using an existing framework. -:mod:`wsgiref` is a reference implementation of the WSGI specification that can +:mod:`!wsgiref` is a reference implementation of the WSGI specification that can be used to add WSGI support to a web server or framework. It provides utilities for manipulating WSGI environment variables and response headers, base classes for implementing WSGI servers, a demo HTTP server that serves WSGI applications, @@ -35,8 +37,8 @@ to tutorials and other resources. .. XXX If you're just trying to write a web application... -:mod:`wsgiref.util` -- WSGI environment utilities -------------------------------------------------- +:mod:`!wsgiref.util` -- WSGI environment utilities +-------------------------------------------------- .. module:: wsgiref.util :synopsis: WSGI environment utilities. @@ -144,7 +146,7 @@ in type annotations. httpd.serve_forever() -In addition to the environment functions above, the :mod:`wsgiref.util` module +In addition to the environment functions above, the :mod:`!wsgiref.util` module also provides these miscellaneous utilities: @@ -184,8 +186,8 @@ also provides these miscellaneous utilities: Support for :meth:`~object.__getitem__` method has been removed. -:mod:`wsgiref.headers` -- WSGI response header tools ----------------------------------------------------- +:mod:`!wsgiref.headers` -- WSGI response header tools +----------------------------------------------------- .. module:: wsgiref.headers :synopsis: WSGI response header tools. @@ -268,8 +270,8 @@ manipulation of WSGI response headers using a mapping-like interface. *headers* parameter is optional. -:mod:`wsgiref.simple_server` -- a simple WSGI HTTP server ---------------------------------------------------------- +:mod:`!wsgiref.simple_server` -- a simple WSGI HTTP server +---------------------------------------------------------- .. module:: wsgiref.simple_server :synopsis: A simple WSGI HTTP server. @@ -310,7 +312,7 @@ request. (E.g., using the :func:`shift_path_info` function from This function is a small but complete WSGI application that returns a text page containing the message "Hello world!" and a list of the key/value pairs provided in the *environ* parameter. It's useful for verifying that a WSGI server (such - as :mod:`wsgiref.simple_server`) is able to run a simple WSGI application + as :mod:`!wsgiref.simple_server`) is able to run a simple WSGI application correctly. The *start_response* callable should follow the :class:`.StartResponse` protocol. @@ -382,8 +384,8 @@ request. (E.g., using the :func:`shift_path_info` function from interface. -:mod:`wsgiref.validate` --- WSGI conformance checker ----------------------------------------------------- +:mod:`!wsgiref.validate` --- WSGI conformance checker +----------------------------------------------------- .. module:: wsgiref.validate :synopsis: WSGI conformance checker. @@ -391,7 +393,7 @@ request. (E.g., using the :func:`shift_path_info` function from When creating new WSGI application objects, frameworks, servers, or middleware, it can be useful to validate the new code's conformance using -:mod:`wsgiref.validate`. This module provides a function that creates WSGI +:mod:`!wsgiref.validate`. This module provides a function that creates WSGI application objects that validate communications between a WSGI server or gateway and a WSGI application object, to check both sides for protocol conformance. @@ -450,8 +452,8 @@ Paste" library. httpd.serve_forever() -:mod:`wsgiref.handlers` -- server/gateway base classes ------------------------------------------------------- +:mod:`!wsgiref.handlers` -- server/gateway base classes +------------------------------------------------------- .. module:: wsgiref.handlers :synopsis: WSGI server/gateway base classes. @@ -622,7 +624,7 @@ input, output, and error streams. The default environment variables to be included in every request's WSGI environment. By default, this is a copy of ``os.environ`` at the time that - :mod:`wsgiref.handlers` was imported, but subclasses can either create their own + :mod:`!wsgiref.handlers` was imported, but subclasses can either create their own at the class or instance level. Note that the dictionary should be considered read-only, since the default value is shared between multiple classes and instances. @@ -773,8 +775,8 @@ input, output, and error streams. .. versionadded:: 3.2 -:mod:`wsgiref.types` -- WSGI types for static type checking ------------------------------------------------------------ +:mod:`!wsgiref.types` -- WSGI types for static type checking +------------------------------------------------------------ .. module:: wsgiref.types :synopsis: WSGI types for static type checking diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 9ffedf7366a..1a5291d018a 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -4,15 +4,11 @@ .. module:: xml.dom.minidom :synopsis: Minimal Document Object Model (DOM) implementation. -.. moduleauthor:: Paul Prescod -.. sectionauthor:: Paul Prescod -.. sectionauthor:: Martin v. Löwis - **Source code:** :source:`Lib/xml/dom/minidom.py` -------------- -:mod:`xml.dom.minidom` is a minimal implementation of the Document Object +:mod:`!xml.dom.minidom` is a minimal implementation of the Document Object Model interface, with an API similar to that in other languages. It is intended to be simpler than the full DOM and also significantly smaller. Users who are not already proficient with the DOM should consider using the @@ -26,7 +22,7 @@ not already proficient with the DOM should consider using the DOM applications typically start by parsing some XML into a DOM. With -:mod:`xml.dom.minidom`, this is done through the parse functions:: +:mod:`!xml.dom.minidom`, this is done through the parse functions:: from xml.dom.minidom import parse, parseString @@ -62,7 +58,7 @@ document. What the :func:`parse` and :func:`parseString` functions do is connect an XML parser with a "DOM builder" that can accept parse events from any SAX parser and -convert them into a DOM tree. The name of the functions are perhaps misleading, +convert them into a DOM tree. The names of the functions are perhaps misleading, but are easy to grasp when learning the interfaces. The parsing of the document will be completed before these functions return; it's simply that these functions do not provide a parser implementation themselves. @@ -70,7 +66,7 @@ functions do not provide a parser implementation themselves. You can also create a :class:`Document` by calling a method on a "DOM Implementation" object. You can get this object either by calling the :func:`getDOMImplementation` function in the :mod:`xml.dom` package or the -:mod:`xml.dom.minidom` module. Once you have a :class:`Document`, you +:mod:`!xml.dom.minidom` module. Once you have a :class:`Document`, you can add child nodes to it to populate the DOM:: from xml.dom.minidom import getDOMImplementation @@ -93,7 +89,7 @@ document: the one that holds all others. Here is an example program:: When you are finished with a DOM tree, you may optionally call the :meth:`unlink` method to encourage early cleanup of the now-unneeded -objects. :meth:`unlink` is an :mod:`xml.dom.minidom`\ -specific +objects. :meth:`unlink` is an :mod:`!xml.dom.minidom`\ -specific extension to the DOM API that renders the node and its descendants essentially useless. Otherwise, Python's garbage collector will eventually take care of the objects in the tree. @@ -101,7 +97,7 @@ eventually take care of the objects in the tree. .. seealso:: `Document Object Model (DOM) Level 1 Specification `_ - The W3C recommendation for the DOM supported by :mod:`xml.dom.minidom`. + The W3C recommendation for the DOM supported by :mod:`!xml.dom.minidom`. .. _minidom-objects: @@ -111,7 +107,7 @@ DOM Objects The definition of the DOM API for Python is given as part of the :mod:`xml.dom` module documentation. This section lists the differences between the API and -:mod:`xml.dom.minidom`. +:mod:`!xml.dom.minidom`. .. method:: Node.unlink() @@ -214,7 +210,7 @@ particular case, we do not take much advantage of the flexibility of the DOM. minidom and the DOM standard ---------------------------- -The :mod:`xml.dom.minidom` module is essentially a DOM 1.0-compatible DOM with +The :mod:`!xml.dom.minidom` module is essentially a DOM 1.0-compatible DOM with some DOM 2 features (primarily namespace features). Usage of the DOM interface in Python is straight-forward. The following mapping @@ -237,7 +233,7 @@ rules apply: * The types ``short int``, ``unsigned int``, ``unsigned long long``, and ``boolean`` all map to Python integer objects. -* The type ``DOMString`` maps to Python strings. :mod:`xml.dom.minidom` supports +* The type ``DOMString`` maps to Python strings. :mod:`!xml.dom.minidom` supports either bytes or strings, but will normally produce strings. Values of type ``DOMString`` may also be ``None`` where allowed to have the IDL ``null`` value by the DOM specification from the W3C. @@ -245,8 +241,8 @@ rules apply: * ``const`` declarations map to variables in their respective scope (e.g. ``xml.dom.minidom.Node.PROCESSING_INSTRUCTION_NODE``); they must not be changed. -* ``DOMException`` is currently not supported in :mod:`xml.dom.minidom`. - Instead, :mod:`xml.dom.minidom` uses standard Python exceptions such as +* ``DOMException`` is currently not supported in :mod:`!xml.dom.minidom`. + Instead, :mod:`!xml.dom.minidom` uses standard Python exceptions such as :exc:`TypeError` and :exc:`AttributeError`. * :class:`NodeList` objects are implemented using Python's built-in list type. @@ -255,7 +251,7 @@ rules apply: however, much more "Pythonic" than the interface defined in the W3C recommendations. -The following interfaces have no implementation in :mod:`xml.dom.minidom`: +The following interfaces have no implementation in :mod:`!xml.dom.minidom`: * :class:`DOMTimeStamp` diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index 8bceeecd463..52340ffe92e 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -4,13 +4,11 @@ .. module:: xml.dom.pulldom :synopsis: Support for building partial DOM trees from SAX events. -.. moduleauthor:: Paul Prescod - **Source code:** :source:`Lib/xml/dom/pulldom.py` -------------- -The :mod:`xml.dom.pulldom` module provides a "pull parser" which can also be +The :mod:`!xml.dom.pulldom` module provides a "pull parser" which can also be asked to produce DOM-accessible fragments of the document where necessary. The basic concept involves pulling "events" from a stream of incoming XML and processing them. In contrast to SAX which also employs an event-driven @@ -74,7 +72,7 @@ given point) or to make use of the :func:`DOMEventStream.expandNode` method and switch to DOM-related processing. -.. class:: PullDom(documentFactory=None) +.. class:: PullDOM(documentFactory=None) Subclass of :class:`xml.sax.handler.ContentHandler`. diff --git a/Doc/library/xml.dom.rst b/Doc/library/xml.dom.rst index f33b19bc272..34e58dcad93 100644 --- a/Doc/library/xml.dom.rst +++ b/Doc/library/xml.dom.rst @@ -4,9 +4,6 @@ .. module:: xml.dom :synopsis: Document Object Model API for Python. -.. sectionauthor:: Paul Prescod -.. sectionauthor:: Martin v. Löwis - **Source code:** :source:`Lib/xml/dom/__init__.py` -------------- @@ -80,7 +77,7 @@ implementations are free to support the strict mapping from IDL). See section Module Contents --------------- -The :mod:`xml.dom` contains the following functions: +The :mod:`!xml.dom` contains the following functions: .. function:: registerDOMImplementation(name, factory) @@ -135,7 +132,7 @@ Some convenience constants are also provided: HyperText Markup Language `_ (section 3.1.1). -In addition, :mod:`xml.dom` contains a base :class:`Node` class and the DOM +In addition, :mod:`!xml.dom` contains a base :class:`Node` class and the DOM exception classes. The :class:`Node` class provided by this module does not implement any of the methods or attributes defined by the DOM specification; concrete DOM implementations must provide those. The :class:`Node` class diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 00075ac2a23..310ccd651e1 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -4,13 +4,11 @@ .. module:: xml.etree.ElementTree :synopsis: Implementation of the ElementTree API. -.. moduleauthor:: Fredrik Lundh - **Source code:** :source:`Lib/xml/etree/ElementTree.py` -------------- -The :mod:`xml.etree.ElementTree` module implements a simple and efficient API +The :mod:`!xml.etree.ElementTree` module implements a simple and efficient API for parsing and creating XML data. .. versionchanged:: 3.3 @@ -28,7 +26,7 @@ for parsing and creating XML data. Tutorial -------- -This is a short tutorial for using :mod:`xml.etree.ElementTree` (``ET`` in +This is a short tutorial for using :mod:`!xml.etree.ElementTree` (``ET`` in short). The goal is to demonstrate some of the building blocks and basic concepts of the module. @@ -656,6 +654,10 @@ Functions .. versionchanged:: 3.13 Added the :meth:`!close` method. + .. versionchanged:: 3.15 + A :exc:`ResourceWarning` is now emitted if the iterator opened a file + and is not explicitly closed. + .. function:: parse(source, parser=None) @@ -689,7 +691,7 @@ Functions .. versionadded:: 3.2 -.. function:: SubElement(parent, tag, attrib={}, **extra) +.. function:: SubElement(parent, tag, /, attrib={}, **extra) Subelement factory. This function creates an element instance, and appends it to an existing element. @@ -700,6 +702,12 @@ Functions attributes. *extra* contains additional attributes, given as keyword arguments. Returns an element instance. + .. versionchanged:: 3.15 + *attrib* can now be a :class:`frozendict`. + + .. versionchanged:: 3.15 + *parent* and *tag* are now positional-only parameters. + .. function:: tostring(element, encoding="us-ascii", method="xml", *, \ xml_declaration=None, default_namespace=None, \ @@ -791,7 +799,7 @@ Here's an example that demonstrates use of the XInclude module. To include an XM By default, the **href** attribute is treated as a file name. You can use custom loaders to override this behaviour. Also note that the standard helper does not support XPointer syntax. -To process this file, load it as usual, and pass the root element to the :mod:`xml.etree.ElementTree` module: +To process this file, load it as usual, and pass the root element to the :mod:`!xml.etree.ElementTree` module: .. code-block:: python @@ -875,7 +883,7 @@ Element Objects :noindex: :no-index: -.. class:: Element(tag, attrib={}, **extra) +.. class:: Element(tag, /, attrib={}, **extra) Element class. This class defines the Element interface, and provides a reference implementation of this interface. @@ -885,6 +893,12 @@ Element Objects an optional dictionary, containing element attributes. *extra* contains additional attributes, given as keyword arguments. + .. versionchanged:: 3.15 + *attrib* can now be a :class:`frozendict`. + + .. versionchanged:: 3.15 + *tag* is now a positional-only parameter. + .. attribute:: tag @@ -1398,10 +1412,10 @@ XMLParser Objects Disabling reparse deferral has security consequences; please see :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` for details. - Note that :meth:`flush` has been backported to some prior releases of - CPython as a security fix. Check for availability of :meth:`flush` - using :func:`hasattr` if used in code running across a variety of Python - versions. + :meth:`!flush` + has been backported to some prior releases of CPython as a security fix. + Check for availability using :func:`hasattr` if used in code running + across a variety of Python versions. .. versionadded:: 3.13 @@ -1476,10 +1490,10 @@ XMLPullParser Objects Disabling reparse deferral has security consequences; please see :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` for details. - Note that :meth:`flush` has been backported to some prior releases of - CPython as a security fix. Check for availability of :meth:`flush` - using :func:`hasattr` if used in code running across a variety of Python - versions. + :meth:`!flush` + has been backported to some prior releases of CPython as a security fix. + Check for availability using :func:`hasattr` if used in code running + across a variety of Python versions. .. versionadded:: 3.13 diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index 28465219a1a..a47d31465b1 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -6,9 +6,6 @@ XML Processing Modules .. module:: xml :synopsis: Package containing XML processing modules -.. sectionauthor:: Christian Heimes -.. sectionauthor:: Georg Brandl - **Source code:** :source:`Lib/xml/` -------------- @@ -20,7 +17,7 @@ Python's interfaces for processing XML are grouped in the ``xml`` package. If you need to parse untrusted or unauthenticated data, see :ref:`xml-security`. -It is important to note that modules in the :mod:`xml` package require that +It is important to note that modules in the :mod:`!xml` package require that there be at least one SAX-compliant XML parser available. The Expat parser is included with Python, so the :mod:`xml.parsers.expat` module will always be available. @@ -53,11 +50,22 @@ XML security An attacker can abuse XML features to carry out denial of service attacks, access local files, generate network connections to other machines, or -circumvent firewalls. +circumvent firewalls when attacker-controlled XML is being parsed, +in Python or elsewhere. -Expat versions lower that 2.6.0 may be vulnerable to "billion laughs", -"quadratic blowup" and "large tokens". Python may be vulnerable if it uses such -older versions of Expat as a system-provided library. +The built-in XML parsers of Python rely on the library `libexpat`_, commonly +called Expat, for parsing XML. + +By default, Expat itself does not access local files or create network +connections. + +Expat versions lower than 2.7.2 may be vulnerable to the "billion laughs", +"quadratic blowup" and "large tokens" vulnerabilities, or to disproportional +use of dynamic memory. +Python bundles a copy of Expat, and whether Python uses the bundled or a +system-wide Expat, depends on how the Python interpreter +:option:`has been configured <--with-system-expat>` in your environment. +Python may be vulnerable if it uses such older versions of Expat. Check :const:`!pyexpat.EXPAT_VERSION`. :mod:`xmlrpc` is **vulnerable** to the "decompression bomb" attack. @@ -90,5 +98,6 @@ large tokens be used to cause denial of service in the application parsing XML. The issue is known as :cve:`2023-52425`. +.. _libexpat: https://github.com/libexpat/libexpat .. _Billion Laughs: https://en.wikipedia.org/wiki/Billion_laughs .. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb diff --git a/Doc/library/xml.sax.handler.rst b/Doc/library/xml.sax.handler.rst index 38ca4507d81..4188debf566 100644 --- a/Doc/library/xml.sax.handler.rst +++ b/Doc/library/xml.sax.handler.rst @@ -4,9 +4,6 @@ .. module:: xml.sax.handler :synopsis: Base classes for SAX event handlers. -.. moduleauthor:: Lars Marius Garshol -.. sectionauthor:: Martin v. Löwis - **Source code:** :source:`Lib/xml/sax/handler.py` -------------- @@ -16,7 +13,7 @@ error handlers, entity resolvers and lexical handlers. Applications normally only need to implement those interfaces whose events they are interested in; they can implement the interfaces in a single object or in multiple objects. Handler implementations should inherit from the base classes provided in the -module :mod:`xml.sax.handler`, so that all methods get default implementations. +module :mod:`!xml.sax.handler`, so that all methods get default implementations. .. class:: ContentHandler @@ -53,7 +50,7 @@ module :mod:`xml.sax.handler`, so that all methods get default implementations. Interface used by the parser to represent low frequency events which may not be of interest to many applications. -In addition to these classes, :mod:`xml.sax.handler` provides symbolic constants +In addition to these classes, :mod:`!xml.sax.handler` provides symbolic constants for the feature and property names. @@ -96,6 +93,14 @@ for the feature and property names. .. data:: feature_external_ges + .. warning:: + + Enabling opens a vulnerability to + `external entity attacks `_ + if the parser is used with user-provided XML content. + Please reflect on your `threat model `_ + before enabling this feature. + | value: ``"http://xml.org/sax/features/external-general-entities"`` | true: Include all external general (text) entities. | false: Do not include external general entities. diff --git a/Doc/library/xml.sax.reader.rst b/Doc/library/xml.sax.reader.rst index b0bc84062e0..1a5ab6a214f 100644 --- a/Doc/library/xml.sax.reader.rst +++ b/Doc/library/xml.sax.reader.rst @@ -4,9 +4,6 @@ .. module:: xml.sax.xmlreader :synopsis: Interface which SAX-compliant XML parsers must implement. -.. moduleauthor:: Lars Marius Garshol -.. sectionauthor:: Martin v. Löwis - **Source code:** :source:`Lib/xml/sax/xmlreader.py` -------------- diff --git a/Doc/library/xml.sax.rst b/Doc/library/xml.sax.rst index 5fa92645a44..77234cac5d9 100644 --- a/Doc/library/xml.sax.rst +++ b/Doc/library/xml.sax.rst @@ -4,15 +4,11 @@ .. module:: xml.sax :synopsis: Package containing SAX2 base classes and convenience functions. -.. moduleauthor:: Lars Marius Garshol -.. sectionauthor:: Fred L. Drake, Jr. -.. sectionauthor:: Martin v. Löwis - **Source code:** :source:`Lib/xml/sax/__init__.py` -------------- -The :mod:`xml.sax` package provides a number of modules which implement the +The :mod:`!xml.sax` package provides a number of modules which implement the Simple API for XML (SAX) interface for Python. The package itself provides the SAX exceptions and the convenience functions which will be most used by users of the SAX API. @@ -89,9 +85,9 @@ module :mod:`xml.sax.xmlreader`. The handler interfaces are defined in :mod:`xml.sax.handler`. For convenience, :class:`~xml.sax.xmlreader.InputSource` (which is often instantiated directly) and the handler classes are also available from -:mod:`xml.sax`. These interfaces are described below. +:mod:`!xml.sax`. These interfaces are described below. -In addition to these classes, :mod:`xml.sax` provides the following exception +In addition to these classes, :mod:`!xml.sax` provides the following exception classes. diff --git a/Doc/library/xml.sax.utils.rst b/Doc/library/xml.sax.utils.rst index 5ee11d58c3d..2de7ae2bda4 100644 --- a/Doc/library/xml.sax.utils.rst +++ b/Doc/library/xml.sax.utils.rst @@ -4,14 +4,11 @@ .. module:: xml.sax.saxutils :synopsis: Convenience functions and classes for use with SAX. -.. moduleauthor:: Lars Marius Garshol -.. sectionauthor:: Martin v. Löwis - **Source code:** :source:`Lib/xml/sax/saxutils.py` -------------- -The module :mod:`xml.sax.saxutils` contains a number of classes and functions +The module :mod:`!xml.sax.saxutils` contains a number of classes and functions that are commonly useful when creating SAX applications, either in direct use, or as base classes. @@ -37,7 +34,7 @@ or as base classes. You can unescape other strings of data by passing a dictionary as the optional *entities* parameter. The keys and values must all be strings; each key will be - replaced with its corresponding value. ``'&'``, ``'<'``, and ``'>'`` + replaced with its corresponding value. ``'&'``, ``'<'``, and ``'>'`` are always unescaped, even if *entities* is provided. diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index 547cb50be78..458e67dbe51 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -4,9 +4,6 @@ .. module:: xmlrpc.client :synopsis: XML-RPC client access. -.. moduleauthor:: Fredrik Lundh -.. sectionauthor:: Eric S. Raymond - **Source code:** :source:`Lib/xmlrpc/client.py` .. XXX Not everything is documented yet. It might be good to describe @@ -23,13 +20,13 @@ between conformable Python objects and XML on the wire. .. warning:: - The :mod:`xmlrpc.client` module is not secure against maliciously + The :mod:`!xmlrpc.client` module is not secure against maliciously constructed data. If you need to parse untrusted or unauthenticated data, see :ref:`xml-security`. .. versionchanged:: 3.5 - For HTTPS URIs, :mod:`xmlrpc.client` now performs all the necessary + For HTTPS URIs, :mod:`!xmlrpc.client` now performs all the necessary certificate and hostname checks by default. .. include:: ../includes/wasm-notavail.rst @@ -179,9 +176,9 @@ ServerProxy Objects A :class:`ServerProxy` instance has a method corresponding to each remote procedure call accepted by the XML-RPC server. Calling the method performs an RPC, dispatched by both name and argument signature (e.g. the same method name -can be overloaded with multiple argument signatures). The RPC finishes by -returning a value, which may be either returned data in a conformant type or a -:class:`Fault` or :class:`ProtocolError` object indicating an error. +can be overloaded with multiple argument signatures). The RPC finishes either +by returning data in a conformant type or by raising a :class:`Fault` or +:class:`ProtocolError` exception indicating an error. Servers that support the XML introspection API support some common methods grouped under the reserved :attr:`~ServerProxy.system` attribute: @@ -275,12 +272,12 @@ DateTime Objects A working example follows. The server code:: - import datetime + import datetime as dt from xmlrpc.server import SimpleXMLRPCServer import xmlrpc.client def today(): - today = datetime.datetime.today() + today = dt.datetime.today() return xmlrpc.client.DateTime(today) server = SimpleXMLRPCServer(("localhost", 8000)) @@ -291,14 +288,14 @@ A working example follows. The server code:: The client code for the preceding server:: import xmlrpc.client - import datetime + import datetime as dt proxy = xmlrpc.client.ServerProxy("http://localhost:8000/") today = proxy.today() - # convert the ISO8601 string to a datetime object - converted = datetime.datetime.strptime(today.value, "%Y%m%dT%H:%M:%S") - print("Today: %s" % converted.strftime("%d.%m.%Y, %H:%M")) + # convert the ISO 8601 string to a datetime object + converted = dt.datetime.strptime(today.value, "%Y%m%dT%H:%M:%S") + print(f"Today: {converted.strftime('%d.%m.%Y, %H:%M')}") .. _binary-objects: @@ -472,7 +469,7 @@ remote server into a single request [#]_. Create an object used to boxcar method calls. *server* is the eventual target of the call. Calls can be made to the result object, but they will immediately - return ``None``, and only store the call name and parameters in the + return ``None``, and only store the call name and arguments in the :class:`MultiCall` object. Calling the object itself causes all stored calls to be transmitted as a single ``system.multicall`` request. The result of this call is a :term:`generator`; iterating over this generator yields the individual @@ -524,7 +521,7 @@ Convenience Functions .. function:: dumps(params, methodname=None, methodresponse=None, encoding=None, allow_none=False) - Convert *params* into an XML-RPC request. or into a response if *methodresponse* + Convert *params* into an XML-RPC request, or into a response if *methodresponse* is true. *params* can be either a tuple of arguments or an instance of the :exc:`Fault` exception class. If *methodresponse* is true, only a single value can be returned, meaning that *params* must be of length 1. *encoding*, if diff --git a/Doc/library/xmlrpc.server.rst b/Doc/library/xmlrpc.server.rst index 2a8f6f8d5fc..5702257dfe2 100644 --- a/Doc/library/xmlrpc.server.rst +++ b/Doc/library/xmlrpc.server.rst @@ -4,14 +4,11 @@ .. module:: xmlrpc.server :synopsis: Basic XML-RPC server implementations. -.. moduleauthor:: Brian Quinlan -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/xmlrpc/server.py` -------------- -The :mod:`xmlrpc.server` module provides a basic server framework for XML-RPC +The :mod:`!xmlrpc.server` module provides a basic server framework for XML-RPC servers written in Python. Servers can either be free standing, using :class:`SimpleXMLRPCServer`, or embedded in a CGI environment, using :class:`CGIXMLRPCRequestHandler`. @@ -19,7 +16,7 @@ servers written in Python. Servers can either be free standing, using .. warning:: - The :mod:`xmlrpc.server` module is not secure against maliciously + The :mod:`!xmlrpc.server` module is not secure against maliciously constructed data. If you need to parse untrusted or unauthenticated data, see :ref:`xml-security`. @@ -72,7 +69,7 @@ servers written in Python. Servers can either be free standing, using .. _simple-xmlrpc-servers: -SimpleXMLRPCServer Objects +SimpleXMLRPCServer objects -------------------------- The :class:`SimpleXMLRPCServer` class is based on @@ -143,7 +140,7 @@ alone XML-RPC servers. .. _simplexmlrpcserver-example: -SimpleXMLRPCServer Example +SimpleXMLRPCServer example ^^^^^^^^^^^^^^^^^^^^^^^^^^ Server code:: @@ -230,11 +227,11 @@ a server allowing dotted names and registering a multicall function. Enabling the *allow_dotted_names* option allows intruders to access your module's global variables and may allow intruders to execute arbitrary code on - your machine. Only use this example only within a secure, closed network. + your machine. Only use this example within a secure, closed network. :: - import datetime + import datetime as dt class ExampleService: def getData(self): @@ -243,7 +240,7 @@ a server allowing dotted names and registering a multicall function. class currentTime: @staticmethod def getCurrentTime(): - return datetime.datetime.now() + return dt.datetime.now() with SimpleXMLRPCServer(("localhost", 8000)) as server: server.register_function(pow) @@ -390,7 +387,7 @@ to HTTP GET requests. Servers can either be free standing, using .. _doc-xmlrpc-servers: -DocXMLRPCServer Objects +DocXMLRPCServer objects ----------------------- The :class:`DocXMLRPCServer` class is derived from :class:`SimpleXMLRPCServer` diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index cdaba07ab46..2083857312f 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -258,7 +258,7 @@ depending on whether your code is written for Python 2 or 3. Creating Standalone Applications with zipapp -------------------------------------------- -Using the :mod:`zipapp` module, it is possible to create self-contained Python +Using the :mod:`!zipapp` module, it is possible to create self-contained Python programs, which can be distributed to end users who only need to have a suitable version of Python installed on their system. The key to doing this is to bundle all of the application's dependencies into the archive, along diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index f6ec33640b6..9999ac26999 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -4,9 +4,6 @@ .. module:: zipfile :synopsis: Read and write ZIP-format archive files. -.. moduleauthor:: James C. Ahlstrom -.. sectionauthor:: James C. Ahlstrom - **Source code:** :source:`Lib/zipfile/` -------------- @@ -16,13 +13,23 @@ provides tools to create, read, write, append, and list a ZIP file. Any advanced use of this module will require an understanding of the format, as defined in `PKZIP Application Note`_. -This module does not currently handle multi-disk ZIP files. +This module does not handle multipart ZIP files. It can handle ZIP files that use the ZIP64 extensions (that is ZIP files that are more than 4 GiB in size). It supports -decryption of encrypted files in ZIP archives, but it currently cannot +decryption of encrypted files in ZIP archives, but it cannot create an encrypted file. Decryption is extremely slow as it is implemented in native Python rather than C. +.. + The following paragraph should be similar to ../includes/optional-module.rst + +Handling compressed archives requires :term:`optional modules ` +such as :mod:`zlib`, :mod:`bz2`, :mod:`lzma`, and :mod:`compression.zstd`. +If any of them are missing from your copy of CPython, +look for documentation from your distributor (that is, +whoever provided Python to you). +If you are the distributor, see :ref:`optional-module-requirements`. + The module defines the following items: .. exception:: BadZipFile @@ -72,7 +79,7 @@ The module defines the following items: Class used to represent information about a member of an archive. Instances of this class are returned by the :meth:`.getinfo` and :meth:`.infolist` - methods of :class:`ZipFile` objects. Most users of the :mod:`zipfile` module + methods of :class:`ZipFile` objects. Most users of the :mod:`!zipfile` module will not need to create these, but only use those created by this module. *filename* should be the full name of the archive member, and *date_time* should be a tuple containing six fields which describe the time @@ -165,7 +172,7 @@ The module defines the following items: .. _zipfile-objects: -ZipFile Objects +ZipFile objects --------------- @@ -199,7 +206,7 @@ ZipFile Objects If *allowZip64* is ``True`` (the default) zipfile will create ZIP files that use the ZIP64 extensions when the zipfile is larger than 4 GiB. If it is - ``false`` :mod:`zipfile` will raise an exception when the ZIP file would + ``false`` :mod:`!zipfile` will raise an exception when the ZIP file would require ZIP64 extensions. The *compresslevel* parameter controls the compression level to use when @@ -238,7 +245,7 @@ ZipFile Objects .. note:: *metadata_encoding* is an instance-wide setting for the ZipFile. - It is not currently possible to set this on a per-member basis. + It is not possible to set this on a per-member basis. This attribute is a workaround for legacy implementations which produce archives with names in the current locale encoding or code page (mostly @@ -526,6 +533,11 @@ ZipFile Objects a closed ZipFile will raise a :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. + .. versionchanged:: 3.14 + Now respects the :envvar:`SOURCE_DATE_EPOCH` environment variable. + If set, it uses this value as the modification timestamp for the file + written into the ZIP archive, instead of using the current time. + .. method:: ZipFile.mkdir(zinfo_or_directory, mode=511) Create a directory inside the archive. If *zinfo_or_directory* is a string, @@ -561,7 +573,7 @@ The following data attributes are also available: .. _path-objects: -Path Objects +Path objects ------------ .. class:: Path(root, at='') @@ -697,7 +709,7 @@ changes. .. _pyzipfile-objects: -PyZipFile Objects +PyZipFile objects ----------------- The :class:`PyZipFile` constructor takes the same parameters as the @@ -774,7 +786,7 @@ The :class:`PyZipFile` constructor takes the same parameters as the .. _zipinfo-objects: -ZipInfo Objects +ZipInfo objects --------------- Instances of the :class:`ZipInfo` class are returned by the :meth:`.getinfo` and @@ -944,10 +956,10 @@ Instances have the following methods and attributes: .. _zipfile-commandline: .. program:: zipfile -Command-Line Interface +Command-line interface ---------------------- -The :mod:`zipfile` module provides a simple command-line interface to interact +The :mod:`!zipfile` module provides a simple command-line interface to interact with ZIP archives. If you want to create a new ZIP archive, specify its name after the :option:`-c` @@ -1019,7 +1031,7 @@ From file itself Decompression may fail due to incorrect password / CRC checksum / ZIP format or unsupported compression method / decryption. -File System limitations +File system limitations ~~~~~~~~~~~~~~~~~~~~~~~ Exceeding limitations on different file systems can cause decompression failed. diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index 765e5cfd3bb..a4136733a55 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -4,15 +4,13 @@ .. module:: zipimport :synopsis: Support for importing Python modules from ZIP archives. -.. moduleauthor:: Just van Rossum - **Source code:** :source:`Lib/zipimport.py` -------------- This module adds the ability to import Python modules (:file:`\*.py`, :file:`\*.pyc`) and packages from ZIP-format archives. It is usually not -needed to use the :mod:`zipimport` module explicitly; it is automatically used +needed to use the :mod:`!zipimport` module explicitly; it is automatically used by the built-in :keyword:`import` mechanism for :data:`sys.path` items that are paths to ZIP archives. @@ -30,7 +28,7 @@ Any files may be present in the ZIP archive, but importers are only invoked for corresponding :file:`.pyc` file, meaning that if a ZIP archive doesn't contain :file:`.pyc` files, importing may be rather slow. -.. versionchanged:: next +.. versionchanged:: 3.15 Zstandard (*zstd*) compressed zip file entries are supported. .. versionchanged:: 3.13 @@ -176,7 +174,7 @@ Examples -------- Here is an example that imports a module from a ZIP archive - note that the -:mod:`zipimport` module is not explicitly used. +:mod:`!zipimport` module is not explicitly used. .. code-block:: shell-session diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index 7c5e9b086e1..f043915c0f4 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -8,15 +8,13 @@ -------------- For applications that require data compression, the functions in this module -allow compression and decompression, using the zlib library. The zlib library -has its own home page at https://www.zlib.net. There are known -incompatibilities between the Python module and versions of the zlib library -earlier than 1.1.3; 1.1.3 has a `security vulnerability `_, so we recommend using -1.1.4 or later. +allow compression and decompression, using the `zlib library `_. + +.. include:: ../includes/optional-module.rst zlib's functions have many options and often need to be used in a particular order. This documentation doesn't attempt to cover all of the permutations; -consult the zlib manual at http://www.zlib.net/manual.html for authoritative +consult the `zlib manual `_ for authoritative information. For reading and writing ``.gz`` files see the :mod:`gzip` module. @@ -56,23 +54,22 @@ The available exception and functions in this module are: that were concurrently computed. To compute checksums sequentially, use :func:`adler32` with the running checksum as the ``value`` argument. - .. versionadded:: next + .. versionadded:: 3.15 -.. function:: compress(data, /, level=-1, wbits=MAX_WBITS) +.. function:: compress(data, /, level=Z_DEFAULT_COMPRESSION, wbits=MAX_WBITS) Compresses the bytes in *data*, returning a bytes object containing compressed data. *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; - ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, ``9`` (Z_BEST_COMPRESSION) - is slowest and produces the most. ``0`` (Z_NO_COMPRESSION) is no compression. - The default value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default - compromise between speed and compression (currently equivalent to level 6). + See :const:`Z_BEST_SPEED` (``1``), :const:`Z_BEST_COMPRESSION` (``9``), + :const:`Z_NO_COMPRESSION` (``0``), and the default, + :const:`Z_DEFAULT_COMPRESSION` (``-1``) for more information about these values. .. _compress-wbits: The *wbits* argument controls the size of the history buffer (or the "window size") used when compressing data, and whether a header and trailer is included in the output. It can take several ranges of values, - defaulting to ``15`` (MAX_WBITS): + defaulting to ``15`` (:const:`MAX_WBITS`): * +9 to +15: The base-two logarithm of the window size, which therefore ranges between 512 and 32768. Larger values produce @@ -96,17 +93,15 @@ The available exception and functions in this module are: The *wbits* parameter is now available to set window bits and compression type. -.. function:: compressobj(level=-1, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY[, zdict]) +.. function:: compressobj(level=Z_DEFAULT_COMPRESSION, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY[, zdict]) Returns a compression object, to be used for compressing data streams that won't fit into memory at once. *level* is the compression level -- an integer from ``0`` to ``9`` or ``-1``. - A value of ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, - while a value of ``9`` (Z_BEST_COMPRESSION) is slowest and produces the most. - ``0`` (Z_NO_COMPRESSION) is no compression. The default value is ``-1`` (Z_DEFAULT_COMPRESSION). - Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression - (currently equivalent to level 6). + See :const:`Z_BEST_SPEED` (``1``), :const:`Z_BEST_COMPRESSION` (``9``), + :const:`Z_NO_COMPRESSION` (``0``), and the default, + :const:`Z_DEFAULT_COMPRESSION` (``-1``) for more information about these values. *method* is the compression algorithm. Currently, the only supported value is :const:`DEFLATED`. @@ -121,7 +116,7 @@ The available exception and functions in this module are: *strategy* is used to tune the compression algorithm. Possible values are :const:`Z_DEFAULT_STRATEGY`, :const:`Z_FILTERED`, :const:`Z_HUFFMAN_ONLY`, - :const:`Z_RLE` (zlib 1.2.0.1) and :const:`Z_FIXED` (zlib 1.2.2.2). + :const:`Z_RLE` and :const:`Z_FIXED`. *zdict* is a predefined compression dictionary. This is a sequence of bytes (such as a :class:`bytes` object) containing subsequences that are expected @@ -162,7 +157,7 @@ The available exception and functions in this module are: that were concurrently computed. To compute checksums sequentially, use :func:`crc32` with the running checksum as the ``value`` argument. - .. versionadded:: next + .. versionadded:: 3.15 .. function:: decompress(data, /, wbits=MAX_WBITS, bufsize=DEF_BUF_SIZE) @@ -249,7 +244,7 @@ Compression objects support the following methods: All pending input is processed, and a bytes object containing the remaining compressed output is returned. *mode* can be selected from the constants :const:`Z_NO_FLUSH`, :const:`Z_PARTIAL_FLUSH`, :const:`Z_SYNC_FLUSH`, - :const:`Z_FULL_FLUSH`, :const:`Z_BLOCK` (zlib 1.2.3.4), or :const:`Z_FINISH`, + :const:`Z_FULL_FLUSH`, :const:`Z_BLOCK`, or :const:`Z_FINISH`, defaulting to :const:`Z_FINISH`. Except :const:`Z_FINISH`, all constants allow compressing further bytestrings of data, while :const:`Z_FINISH` finishes the compressed stream and prevents compressing any more data. After calling :meth:`flush` @@ -313,6 +308,11 @@ Decompression objects support the following methods and attributes: :attr:`unconsumed_tail`. This bytestring must be passed to a subsequent call to :meth:`decompress` if decompression is to continue. If *max_length* is zero then the whole input is decompressed, and :attr:`unconsumed_tail` is empty. + For example, the full content could be read like:: + + process_output(d.decompress(data, max_length)) + while chunk := d.decompress(d.unconsumed_tail, max_length): + process_output(chunk) .. versionchanged:: 3.6 *max_length* can be used as a keyword argument. @@ -340,6 +340,137 @@ Decompression objects support the following methods and attributes: objects. +The following constants are available to configure compression and decompression +behavior: + +.. data:: DEFLATED + + The deflate compression method. + + +.. data:: MAX_WBITS + + The maximum window size, expressed as a power of 2. + For example, if :const:`!MAX_WBITS` is ``15`` it results in a window size + of ``32 KiB``. + + +.. data:: DEF_MEM_LEVEL + + The default memory level for compression objects. + + +.. data:: DEF_BUF_SIZE + + The default buffer size for decompression operations. + + +.. data:: Z_NO_COMPRESSION + + Compression level ``0``; no compression. + + .. versionadded:: 3.6 + + +.. data:: Z_BEST_SPEED + + Compression level ``1``; fastest and produces the least compression. + + +.. data:: Z_BEST_COMPRESSION + + Compression level ``9``; slowest and produces the most compression. + + +.. data:: Z_DEFAULT_COMPRESSION + + Default compression level (``-1``); a compromise between speed and + compression. Currently equivalent to compression level ``6``. + + +.. data:: Z_DEFAULT_STRATEGY + + Default compression strategy, for normal data. + + +.. data:: Z_FILTERED + + Compression strategy for data produced by a filter (or predictor). + + +.. data:: Z_HUFFMAN_ONLY + + Compression strategy that forces Huffman coding only. + + +.. data:: Z_RLE + + Compression strategy that limits match distances to one (run-length encoding). + + This constant is only available if Python was compiled with zlib + 1.2.0.1 or greater. + + .. versionadded:: 3.6 + + +.. data:: Z_FIXED + + Compression strategy that prevents the use of dynamic Huffman codes. + + This constant is only available if Python was compiled with zlib + 1.2.2.2 or greater. + + .. versionadded:: 3.6 + + +.. data:: Z_NO_FLUSH + + Flush mode ``0``. No special flushing behavior. + + .. versionadded:: 3.6 + + +.. data:: Z_PARTIAL_FLUSH + + Flush mode ``1``. Flush as much output as possible. + + +.. data:: Z_SYNC_FLUSH + + Flush mode ``2``. All output is flushed and the output is aligned to a byte boundary. + + +.. data:: Z_FULL_FLUSH + + Flush mode ``3``. All output is flushed and the compression state is reset. + + +.. data:: Z_FINISH + + Flush mode ``4``. All pending input is processed, no more input is expected. + + +.. data:: Z_BLOCK + + Flush mode ``5``. A deflate block is completed and emitted. + + This constant is only available if Python was compiled with zlib + 1.2.2.2 or greater. + + .. versionadded:: 3.6 + + +.. data:: Z_TREES + + Flush mode ``6``, for inflate operations. Instructs inflate to return when + it gets to the next deflate block boundary. + + This constant is only available if Python was compiled with zlib + 1.2.3.4 or greater. + + .. versionadded:: 3.6 + + Information about the version of the zlib library in use is available through the following constants: @@ -375,10 +506,10 @@ the following constants: Module :mod:`gzip` Reading and writing :program:`gzip`\ -format files. - http://www.zlib.net + https://www.zlib.net The zlib library home page. - http://www.zlib.net/manual.html + https://www.zlib.net/manual.html The zlib manual explains the semantics and usage of the library's many functions. diff --git a/Doc/library/zoneinfo.rst b/Doc/library/zoneinfo.rst index 53d8e2598ec..f5d3ade478f 100644 --- a/Doc/library/zoneinfo.rst +++ b/Doc/library/zoneinfo.rst @@ -6,16 +6,13 @@ .. versionadded:: 3.9 -.. moduleauthor:: Paul Ganssle -.. sectionauthor:: Paul Ganssle - **Source code:** :source:`Lib/zoneinfo` -------------- -The :mod:`zoneinfo` module provides a concrete time zone implementation to +The :mod:`!zoneinfo` module provides a concrete time zone implementation to support the IANA time zone database as originally specified in :pep:`615`. By -default, :mod:`zoneinfo` uses the system's time zone data if available; if no +default, :mod:`!zoneinfo` uses the system's time zone data if available; if no system time zone data is available, the library will fall back to using the first-party :pypi:`tzdata` package available on PyPI. @@ -40,24 +37,24 @@ the constructor, the :meth:`datetime.replace ` method or :meth:`datetime.astimezone `:: >>> from zoneinfo import ZoneInfo - >>> from datetime import datetime, timedelta + >>> import datetime as dt - >>> dt = datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) - >>> print(dt) + >>> when = dt.datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) + >>> print(when) 2020-10-31 12:00:00-07:00 - >>> dt.tzname() + >>> when.tzname() 'PDT' Datetimes constructed in this way are compatible with datetime arithmetic and handle daylight saving time transitions with no further intervention:: - >>> dt_add = dt + timedelta(days=1) + >>> when_add = when + dt.timedelta(days=1) - >>> print(dt_add) + >>> print(when_add) 2020-11-01 12:00:00-08:00 - >>> dt_add.tzname() + >>> when_add.tzname() 'PST' These time zones also support the :attr:`~datetime.datetime.fold` attribute @@ -66,26 +63,25 @@ times (such as a daylight saving time to standard time transition), the offset from *before* the transition is used when ``fold=0``, and the offset *after* the transition is used when ``fold=1``, for example:: - >>> dt = datetime(2020, 11, 1, 1, tzinfo=ZoneInfo("America/Los_Angeles")) - >>> print(dt) + >>> when = dt.datetime(2020, 11, 1, 1, tzinfo=ZoneInfo("America/Los_Angeles")) + >>> print(when) 2020-11-01 01:00:00-07:00 - >>> print(dt.replace(fold=1)) + >>> print(when.replace(fold=1)) 2020-11-01 01:00:00-08:00 When converting from another time zone, the fold will be set to the correct value:: - >>> from datetime import timezone >>> LOS_ANGELES = ZoneInfo("America/Los_Angeles") - >>> dt_utc = datetime(2020, 11, 1, 8, tzinfo=timezone.utc) + >>> when_utc = dt.datetime(2020, 11, 1, 8, tzinfo=dt.timezone.utc) >>> # Before the PDT -> PST transition - >>> print(dt_utc.astimezone(LOS_ANGELES)) + >>> print(when_utc.astimezone(LOS_ANGELES)) 2020-11-01 01:00:00-07:00 >>> # After the PDT -> PST transition - >>> print((dt_utc + timedelta(hours=1)).astimezone(LOS_ANGELES)) + >>> print((when_utc + dt.timedelta(hours=1)).astimezone(LOS_ANGELES)) 2020-11-01 01:00:00-08:00 Data sources @@ -206,6 +202,9 @@ The ``ZoneInfo`` class has two alternate constructors: Objects created via this constructor cannot be pickled (see `pickling`_). + :exc:`ValueError` is raised if the data read from *file_obj* is not a valid + TZif file. + .. classmethod:: ZoneInfo.no_cache(key) An alternate constructor that bypasses the constructor's cache. It is @@ -276,8 +275,8 @@ the note on usage in the attribute documentation):: >>> str(zone) 'Pacific/Kwajalein' - >>> dt = datetime(2020, 4, 1, 3, 15, tzinfo=zone) - >>> f"{dt.isoformat()} [{dt.tzinfo}]" + >>> when = dt.datetime(2020, 4, 1, 3, 15, tzinfo=zone) + >>> f"{when.isoformat()} [{when.tzinfo}]" '2020-04-01T03:15:00+12:00 [Pacific/Kwajalein]' For objects constructed from a file without specifying a ``key`` parameter, @@ -299,7 +298,7 @@ The behavior of a ``ZoneInfo`` file depends on how it was constructed: 1. ``ZoneInfo(key)``: When constructed with the primary constructor, a ``ZoneInfo`` object is serialized by key, and when deserialized, the deserializing process uses the primary and thus it is expected that these - are expected to be the same object as other references to the same time + are the same object as other references to the same time zone. For example, if ``europe_berlin_pkl`` is a string containing a pickle constructed from ``ZoneInfo("Europe/Berlin")``, one would expect the following behavior: @@ -349,7 +348,7 @@ Functions This function only includes canonical zone names and does not include "special" zones such as those under the ``posix/`` and ``right/`` - directories, or the ``posixrules`` zone. + directories, the ``posixrules`` or the ``localtime`` zone. .. caution:: diff --git a/Doc/license.rst b/Doc/license.rst index 269d14cefbc..23860815e7c 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -848,29 +848,10 @@ expat ----- The :mod:`pyexpat ` extension is built using an included copy of the expat -sources unless the build is configured ``--with-system-expat``:: +sources unless the build is configured :option:`--with-system-expat`: - Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd - and Clark Cooper - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +.. literalinclude:: ../Modules/expat/COPYING + :language: text libffi @@ -1199,3 +1180,39 @@ The d3.js library contains the following notice:: OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + +Pixi packages +------------- + +The Pixi package definitions found in :file:`Tools/pixi-packages` are derived +from https://github.com/conda-forge/python-feedstock which contains the following +license:: + + BSD-3-Clause license + Copyright (c) 2015-2026, conda-forge contributors + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + DAMAGE. diff --git a/Doc/pylock.toml b/Doc/pylock.toml new file mode 100644 index 00000000000..f1febe21c23 --- /dev/null +++ b/Doc/pylock.toml @@ -0,0 +1,256 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile Doc/requirements.txt --exclude-newer P14D --exclude-newer-package linklint=PT0S --exclude-newer-package python-docs-theme=PT0S --no-cache --output-file Doc/pylock.toml --python-version 3.12 --universal +lock-version = "1.0" +created-by = "uv" + +[[packages]] +name = "alabaster" +version = "1.0.0" +sdist = { url = "https://files.pythonhosted.org/packages/a6/f8/d9c74d0daf3f742840fd818d69cfae176fa332022fd44e3469487d5a9420/alabaster-1.0.0.tar.gz", upload-time = 2024-07-26T18:15:03Z, size = 24210, hashes = { sha256 = "c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", upload-time = 2024-07-26T18:15:02Z, size = 13929, hashes = { sha256 = "fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b" } }] + +[[packages]] +name = "babel" +version = "2.18.0" +sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", upload-time = 2026-02-01T12:30:56Z, size = 9959554, hashes = { sha256 = "b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", upload-time = 2026-02-01T12:30:53Z, size = 10196845, hashes = { sha256 = "e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35" } }] + +[[packages]] +name = "blurb" +version = "2.0.0" +sdist = { url = "https://files.pythonhosted.org/packages/d7/82/8597d891f4b03f3eaefcb4213a811643d558350cac9a69864d127832cc4f/blurb-2.0.0.tar.gz", upload-time = 2025-01-15T12:48:53Z, size = 24666, hashes = { sha256 = "c78d8114294225a4f7a2eabba6e05d36a6a50e45ba9f5a41afabc198350038e0" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/b4/03/374bd9e31b58e8a8e5dc65cc3f68ca7cdd716c32b5e5dcb0e1b76bb75b4a/blurb-2.0.0-py3-none-any.whl", upload-time = 2025-01-15T12:48:49Z, size = 18924, hashes = { sha256 = "f6d0e858dbe94765f6a89b8228217ffdb9c19cff08fc8f2c3153954846d31aa1" } }] + +[[packages]] +name = "certifi" +version = "2026.2.25" +sdist = { url = "https://files.pythonhosted.org/packages/af/2d/7bf41579a8986e348fa033a31cdd0e4121114f6bce2457e8876010b092dd/certifi-2026.2.25.tar.gz", upload-time = 2026-02-25T02:54:17Z, size = 155029, hashes = { sha256 = "e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", upload-time = 2026-02-25T02:54:15Z, size = 153684, hashes = { sha256 = "027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa" } }] + +[[packages]] +name = "charset-normalizer" +version = "3.4.7" +sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", upload-time = 2026-04-02T09:28:39Z, size = 144271, hashes = { sha256 = "ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5" } } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", upload-time = 2026-04-02T09:26:24Z, size = 311328, hashes = { sha256 = "eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46" } }, + { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", upload-time = 2026-04-02T09:26:25Z, size = 208061, hashes = { sha256 = "6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2" } }, + { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", upload-time = 2026-04-02T09:26:26Z, size = 229031, hashes = { sha256 = "e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b" } }, + { url = "https://files.pythonhosted.org/packages/dc/67/675a46eb016118a2fbde5a277a5d15f4f69d5f3f5f338e5ee2f8948fcf43/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", upload-time = 2026-04-02T09:26:28Z, size = 225239, hashes = { sha256 = "edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a" } }, + { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", upload-time = 2026-04-02T09:26:29Z, size = 216589, hashes = { sha256 = "5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116" } }, + { url = "https://files.pythonhosted.org/packages/b1/f1/6d2b0b261b6c4ceef0fcb0d17a01cc5bc53586c2d4796fa04b5c540bc13d/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", upload-time = 2026-04-02T09:26:30Z, size = 202733, hashes = { sha256 = "203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb" } }, + { url = "https://files.pythonhosted.org/packages/6f/c0/7b1f943f7e87cc3db9626ba17807d042c38645f0a1d4415c7a14afb5591f/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", upload-time = 2026-04-02T09:26:31Z, size = 212652, hashes = { sha256 = "298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1" } }, + { url = "https://files.pythonhosted.org/packages/38/dd/5a9ab159fe45c6e72079398f277b7d2b523e7f716acc489726115a910097/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", upload-time = 2026-04-02T09:26:33Z, size = 211229, hashes = { sha256 = "708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15" } }, + { url = "https://files.pythonhosted.org/packages/d5/ff/531a1cad5ca855d1c1a8b69cb71abfd6d85c0291580146fda7c82857caa1/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", upload-time = 2026-04-02T09:26:34Z, size = 203552, hashes = { sha256 = "0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5" } }, + { url = "https://files.pythonhosted.org/packages/c1/4c/a5fb52d528a8ca41f7598cb619409ece30a169fbdf9cdce592e53b46c3a6/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", upload-time = 2026-04-02T09:26:36Z, size = 230806, hashes = { sha256 = "4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d" } }, + { url = "https://files.pythonhosted.org/packages/59/7a/071feed8124111a32b316b33ae4de83d36923039ef8cf48120266844285b/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", upload-time = 2026-04-02T09:26:37Z, size = 212316, hashes = { sha256 = "aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7" } }, + { url = "https://files.pythonhosted.org/packages/fd/35/f7dba3994312d7ba508e041eaac39a36b120f32d4c8662b8814dab876431/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", upload-time = 2026-04-02T09:26:38Z, size = 227274, hashes = { sha256 = "fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464" } }, + { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", upload-time = 2026-04-02T09:26:40Z, size = 218468, hashes = { sha256 = "bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49" } }, + { url = "https://files.pythonhosted.org/packages/86/eb/890922a8b03a568ca2f336c36585a4713c55d4d67bf0f0c78924be6315ca/charset_normalizer-3.4.7-cp312-cp312-win32.whl", upload-time = 2026-04-02T09:26:41Z, size = 148460, hashes = { sha256 = "2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c" } }, + { url = "https://files.pythonhosted.org/packages/35/d9/0e7dffa06c5ab081f75b1b786f0aefc88365825dfcd0ac544bdb7b2b6853/charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", upload-time = 2026-04-02T09:26:42Z, size = 159330, hashes = { sha256 = "5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6" } }, + { url = "https://files.pythonhosted.org/packages/9e/5d/481bcc2a7c88ea6b0878c299547843b2521ccbc40980cb406267088bc701/charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", upload-time = 2026-04-02T09:26:44Z, size = 147828, hashes = { sha256 = "56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d" } }, + { url = "https://files.pythonhosted.org/packages/c1/3b/66777e39d3ae1ddc77ee606be4ec6d8cbd4c801f65e5a1b6f2b11b8346dd/charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", upload-time = 2026-04-02T09:26:45Z, size = 309627, hashes = { sha256 = "f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063" } }, + { url = "https://files.pythonhosted.org/packages/2e/4e/b7f84e617b4854ade48a1b7915c8ccfadeba444d2a18c291f696e37f0d3b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", upload-time = 2026-04-02T09:26:46Z, size = 207008, hashes = { sha256 = "0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c" } }, + { url = "https://files.pythonhosted.org/packages/c4/bb/ec73c0257c9e11b268f018f068f5d00aa0ef8c8b09f7753ebd5f2880e248/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", upload-time = 2026-04-02T09:26:48Z, size = 228303, hashes = { sha256 = "a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66" } }, + { url = "https://files.pythonhosted.org/packages/85/fb/32d1f5033484494619f701e719429c69b766bfc4dbc61aa9e9c8c166528b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", upload-time = 2026-04-02T09:26:49Z, size = 224282, hashes = { sha256 = "3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18" } }, + { url = "https://files.pythonhosted.org/packages/fa/07/330e3a0dda4c404d6da83b327270906e9654a24f6c546dc886a0eb0ffb23/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", upload-time = 2026-04-02T09:26:50Z, size = 215595, hashes = { sha256 = "e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd" } }, + { url = "https://files.pythonhosted.org/packages/e3/7c/fc890655786e423f02556e0216d4b8c6bcb6bdfa890160dc66bf52dee468/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", upload-time = 2026-04-02T09:26:52Z, size = 201986, hashes = { sha256 = "f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215" } }, + { url = "https://files.pythonhosted.org/packages/d8/97/bfb18b3db2aed3b90cf54dc292ad79fdd5ad65c4eae454099475cbeadd0d/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", upload-time = 2026-04-02T09:26:53Z, size = 211711, hashes = { sha256 = "e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859" } }, + { url = "https://files.pythonhosted.org/packages/6f/a5/a581c13798546a7fd557c82614a5c65a13df2157e9ad6373166d2a3e645d/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", upload-time = 2026-04-02T09:26:54Z, size = 210036, hashes = { sha256 = "7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8" } }, + { url = "https://files.pythonhosted.org/packages/8c/bf/b3ab5bcb478e4193d517644b0fb2bf5497fbceeaa7a1bc0f4d5b50953861/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", upload-time = 2026-04-02T09:26:56Z, size = 202998, hashes = { sha256 = "481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5" } }, + { url = "https://files.pythonhosted.org/packages/e7/4e/23efd79b65d314fa320ec6017b4b5834d5c12a58ba4610aa353af2e2f577/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", upload-time = 2026-04-02T09:26:57Z, size = 230056, hashes = { sha256 = "f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832" } }, + { url = "https://files.pythonhosted.org/packages/b9/9f/1e1941bc3f0e01df116e68dc37a55c4d249df5e6fa77f008841aef68264f/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", upload-time = 2026-04-02T09:26:58Z, size = 211537, hashes = { sha256 = "f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6" } }, + { url = "https://files.pythonhosted.org/packages/80/0f/088cbb3020d44428964a6c97fe1edfb1b9550396bf6d278330281e8b709c/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", upload-time = 2026-04-02T09:27:00Z, size = 226176, hashes = { sha256 = "3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48" } }, + { url = "https://files.pythonhosted.org/packages/6a/9f/130394f9bbe06f4f63e22641d32fc9b202b7e251c9aef4db044324dac493/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", upload-time = 2026-04-02T09:27:02Z, size = 217723, hashes = { sha256 = "64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a" } }, + { url = "https://files.pythonhosted.org/packages/73/55/c469897448a06e49f8fa03f6caae97074fde823f432a98f979cc42b90e69/charset_normalizer-3.4.7-cp313-cp313-win32.whl", upload-time = 2026-04-02T09:27:03Z, size = 148085, hashes = { sha256 = "4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e" } }, + { url = "https://files.pythonhosted.org/packages/5d/78/1b74c5bbb3f99b77a1715c91b3e0b5bdb6fe302d95ace4f5b1bec37b0167/charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", upload-time = 2026-04-02T09:27:04Z, size = 158819, hashes = { sha256 = "3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110" } }, + { url = "https://files.pythonhosted.org/packages/68/86/46bd42279d323deb8687c4a5a811fd548cb7d1de10cf6535d099877a9a9f/charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", upload-time = 2026-04-02T09:27:05Z, size = 147915, hashes = { sha256 = "80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b" } }, + { url = "https://files.pythonhosted.org/packages/97/c8/c67cb8c70e19ef1960b97b22ed2a1567711de46c4ddf19799923adc836c2/charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", upload-time = 2026-04-02T09:27:07Z, size = 309234, hashes = { sha256 = "c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0" } }, + { url = "https://files.pythonhosted.org/packages/99/85/c091fdee33f20de70d6c8b522743b6f831a2f1cd3ff86de4c6a827c48a76/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", upload-time = 2026-04-02T09:27:08Z, size = 208042, hashes = { sha256 = "1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a" } }, + { url = "https://files.pythonhosted.org/packages/87/1c/ab2ce611b984d2fd5d86a5a8a19c1ae26acac6bad967da4967562c75114d/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", upload-time = 2026-04-02T09:27:09Z, size = 228706, hashes = { sha256 = "54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b" } }, + { url = "https://files.pythonhosted.org/packages/a8/29/2b1d2cb00bf085f59d29eb773ce58ec2d325430f8c216804a0a5cd83cbca/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", upload-time = 2026-04-02T09:27:11Z, size = 224727, hashes = { sha256 = "715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41" } }, + { url = "https://files.pythonhosted.org/packages/47/5c/032c2d5a07fe4d4855fea851209cca2b6f03ebeb6d4e3afdb3358386a684/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", upload-time = 2026-04-02T09:27:12Z, size = 215882, hashes = { sha256 = "bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e" } }, + { url = "https://files.pythonhosted.org/packages/2c/c2/356065d5a8b78ed04499cae5f339f091946a6a74f91e03476c33f0ab7100/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", upload-time = 2026-04-02T09:27:13Z, size = 200860, hashes = { sha256 = "c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae" } }, + { url = "https://files.pythonhosted.org/packages/0c/cd/a32a84217ced5039f53b29f460962abb2d4420def55afabe45b1c3c7483d/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", upload-time = 2026-04-02T09:27:15Z, size = 211564, hashes = { sha256 = "3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18" } }, + { url = "https://files.pythonhosted.org/packages/44/86/58e6f13ce26cc3b8f4a36b94a0f22ae2f00a72534520f4ae6857c4b81f89/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", upload-time = 2026-04-02T09:27:16Z, size = 211276, hashes = { sha256 = "e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b" } }, + { url = "https://files.pythonhosted.org/packages/8f/fe/d17c32dc72e17e155e06883efa84514ca375f8a528ba2546bee73fc4df81/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", upload-time = 2026-04-02T09:27:18Z, size = 201238, hashes = { sha256 = "a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356" } }, + { url = "https://files.pythonhosted.org/packages/6a/29/f33daa50b06525a237451cdb6c69da366c381a3dadcd833fa5676bc468b3/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", upload-time = 2026-04-02T09:27:19Z, size = 230189, hashes = { sha256 = "2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab" } }, + { url = "https://files.pythonhosted.org/packages/b6/6e/52c84015394a6a0bdcd435210a7e944c5f94ea1055f5cc5d56c5fe368e7b/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", upload-time = 2026-04-02T09:27:20Z, size = 211352, hashes = { sha256 = "e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46" } }, + { url = "https://files.pythonhosted.org/packages/8c/d7/4353be581b373033fb9198bf1da3cf8f09c1082561e8e922aa7b39bf9fe8/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", upload-time = 2026-04-02T09:27:22Z, size = 227024, hashes = { sha256 = "d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44" } }, + { url = "https://files.pythonhosted.org/packages/30/45/99d18aa925bd1740098ccd3060e238e21115fffbfdcb8f3ece837d0ace6c/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", upload-time = 2026-04-02T09:27:23Z, size = 217869, hashes = { sha256 = "7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72" } }, + { url = "https://files.pythonhosted.org/packages/5c/05/5ee478aa53f4bb7996482153d4bfe1b89e0f087f0ab6b294fcf92d595873/charset_normalizer-3.4.7-cp314-cp314-win32.whl", upload-time = 2026-04-02T09:27:25Z, size = 148541, hashes = { sha256 = "5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10" } }, + { url = "https://files.pythonhosted.org/packages/48/77/72dcb0921b2ce86420b2d79d454c7022bf5be40202a2a07906b9f2a35c97/charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", upload-time = 2026-04-02T09:27:26Z, size = 159634, hashes = { sha256 = "92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f" } }, + { url = "https://files.pythonhosted.org/packages/c6/a3/c2369911cd72f02386e4e340770f6e158c7980267da16af8f668217abaa0/charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", upload-time = 2026-04-02T09:27:28Z, size = 148384, hashes = { sha256 = "67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246" } }, + { url = "https://files.pythonhosted.org/packages/94/09/7e8a7f73d24dba1f0035fbbf014d2c36828fc1bf9c88f84093e57d315935/charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", upload-time = 2026-04-02T09:27:29Z, size = 330133, hashes = { sha256 = "effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24" } }, + { url = "https://files.pythonhosted.org/packages/8d/da/96975ddb11f8e977f706f45cddd8540fd8242f71ecdb5d18a80723dcf62c/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", upload-time = 2026-04-02T09:27:30Z, size = 216257, hashes = { sha256 = "fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79" } }, + { url = "https://files.pythonhosted.org/packages/e5/e8/1d63bf8ef2d388e95c64b2098f45f84758f6d102a087552da1485912637b/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", upload-time = 2026-04-02T09:27:32Z, size = 234851, hashes = { sha256 = "733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960" } }, + { url = "https://files.pythonhosted.org/packages/9b/40/e5ff04233e70da2681fa43969ad6f66ca5611d7e669be0246c4c7aaf6dc8/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", upload-time = 2026-04-02T09:27:34Z, size = 233393, hashes = { sha256 = "a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4" } }, + { url = "https://files.pythonhosted.org/packages/be/c1/06c6c49d5a5450f76899992f1ee40b41d076aee9279b49cf9974d2f313d5/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", upload-time = 2026-04-02T09:27:35Z, size = 223251, hashes = { sha256 = "6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e" } }, + { url = "https://files.pythonhosted.org/packages/2b/9f/f2ff16fb050946169e3e1f82134d107e5d4ae72647ec8a1b1446c148480f/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", upload-time = 2026-04-02T09:27:36Z, size = 206609, hashes = { sha256 = "a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1" } }, + { url = "https://files.pythonhosted.org/packages/69/d5/a527c0cd8d64d2eab7459784fb4169a0ac76e5a6fc5237337982fd61347e/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", upload-time = 2026-04-02T09:27:38Z, size = 220014, hashes = { sha256 = "3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44" } }, + { url = "https://files.pythonhosted.org/packages/7e/80/8a7b8104a3e203074dc9aa2c613d4b726c0e136bad1cc734594b02867972/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", upload-time = 2026-04-02T09:27:39Z, size = 218979, hashes = { sha256 = "8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e" } }, + { url = "https://files.pythonhosted.org/packages/02/9a/b759b503d507f375b2b5c153e4d2ee0a75aa215b7f2489cf314f4541f2c0/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", upload-time = 2026-04-02T09:27:40Z, size = 209238, hashes = { sha256 = "cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3" } }, + { url = "https://files.pythonhosted.org/packages/c2/4e/0f3f5d47b86bdb79256e7290b26ac847a2832d9a4033f7eb2cd4bcf4bb5b/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", upload-time = 2026-04-02T09:27:42Z, size = 236110, hashes = { sha256 = "0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0" } }, + { url = "https://files.pythonhosted.org/packages/96/23/bce28734eb3ed2c91dcf93abeb8a5cf393a7b2749725030bb630e554fdd8/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", upload-time = 2026-04-02T09:27:43Z, size = 219824, hashes = { sha256 = "752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e" } }, + { url = "https://files.pythonhosted.org/packages/2c/6f/6e897c6984cc4d41af319b077f2f600fc8214eb2fe2d6bcb79141b882400/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", upload-time = 2026-04-02T09:27:45Z, size = 233103, hashes = { sha256 = "8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb" } }, + { url = "https://files.pythonhosted.org/packages/76/22/ef7bd0fe480a0ae9b656189ec00744b60933f68b4f42a7bb06589f6f576a/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", upload-time = 2026-04-02T09:27:46Z, size = 225194, hashes = { sha256 = "ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe" } }, + { url = "https://files.pythonhosted.org/packages/c5/a7/0e0ab3e0b5bc1219bd80a6a0d4d72ca74d9250cb2382b7c699c147e06017/charset_normalizer-3.4.7-cp314-cp314t-win32.whl", upload-time = 2026-04-02T09:27:48Z, size = 159827, hashes = { sha256 = "c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0" } }, + { url = "https://files.pythonhosted.org/packages/7a/1d/29d32e0fb40864b1f878c7f5a0b343ae676c6e2b271a2d55cc3a152391da/charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", upload-time = 2026-04-02T09:27:49Z, size = 174168, hashes = { sha256 = "03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c" } }, + { url = "https://files.pythonhosted.org/packages/de/32/d92444ad05c7a6e41fb2036749777c163baf7a0301a040cb672d6b2b1ae9/charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", upload-time = 2026-04-02T09:27:51Z, size = 153018, hashes = { sha256 = "c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d" } }, + { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", upload-time = 2026-04-02T09:28:37Z, size = 61958, hashes = { sha256 = "3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d" } }, +] + +[[packages]] +name = "colorama" +version = "0.4.6" +marker = "sys_platform == 'win32'" +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", upload-time = 2022-10-25T02:36:22Z, size = 27697, hashes = { sha256 = "08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", upload-time = 2022-10-25T02:36:20Z, size = 25335, hashes = { sha256 = "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" } }] + +[[packages]] +name = "docutils" +version = "0.21.2" +sdist = { url = "https://files.pythonhosted.org/packages/ae/ed/aefcc8cd0ba62a0560c3c18c33925362d46c6075480bfa4df87b28e169a9/docutils-0.21.2.tar.gz", upload-time = 2024-04-23T18:57:18Z, size = 2204444, hashes = { sha256 = "3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", upload-time = 2024-04-23T18:57:14Z, size = 587408, hashes = { sha256 = "dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2" } }] + +[[packages]] +name = "idna" +version = "3.11" +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", upload-time = 2025-10-12T14:55:20Z, size = 194582, hashes = { sha256 = "795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", upload-time = 2025-10-12T14:55:18Z, size = 71008, hashes = { sha256 = "771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea" } }] + +[[packages]] +name = "imagesize" +version = "1.5.0" +sdist = { url = "https://files.pythonhosted.org/packages/cf/59/4b0dd64676aa6fb4986a755790cb6fc558559cf0084effad516820208ec3/imagesize-1.5.0.tar.gz", upload-time = 2026-03-03T01:59:54Z, size = 1281127, hashes = { sha256 = "8bfc5363a7f2133a89f0098451e0bcb1cd71aba4dc02bbcecb39d99d40e1b94f" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/1e/b1/a0662b03103c66cf77101a187f396ea91167cd9b7d5d3a2e465ad2c7ee9b/imagesize-1.5.0-py2.py3-none-any.whl", upload-time = 2026-03-03T01:59:52Z, size = 5763, hashes = { sha256 = "32677681b3f434c2cb496f00e89c5a291247b35b1f527589909e008057da5899" } }] + +[[packages]] +name = "jinja2" +version = "3.1.6" +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", upload-time = 2025-03-05T20:05:02Z, size = 245115, hashes = { sha256 = "0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", upload-time = 2025-03-05T20:05:00Z, size = 134899, hashes = { sha256 = "85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67" } }] + +[[packages]] +name = "linklint" +version = "0.4.1" +sdist = { url = "https://files.pythonhosted.org/packages/61/bc/9972ace8643a04a74210942717fd20c1c34d96079b59fd7790b4db56df7d/linklint-0.4.1.tar.gz", upload-time = 2026-03-27T10:48:40Z, size = 20588, hashes = { sha256 = "a5d291a0d8a7ab8b1f96f62bb7e1d9d2c79d8eceb934e2efc0235d6b2e77f19b" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/af/88/9c4865cdbd6f73fff668706072c421a329de79c3b69e0aa511679a2ff4f3/linklint-0.4.1-py3-none-any.whl", upload-time = 2026-03-27T10:48:38Z, size = 12186, hashes = { sha256 = "78ff4d23ff3d3c62837fa34f0dcb909593dea52a2a1f426307264f081a8b41b5" } }] + +[[packages]] +name = "markupsafe" +version = "2.1.5" +sdist = { url = "https://files.pythonhosted.org/packages/87/5b/aae44c6655f3801e81aa3eef09dbbf012431987ba564d7231722f68df02d/MarkupSafe-2.1.5.tar.gz", upload-time = 2024-02-02T16:31:22Z, size = 19384, hashes = { sha256 = "d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b" } } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/bd/583bf3e4c8d6a321938c13f49d44024dbe5ed63e0a7ba127e454a66da974/MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", upload-time = 2024-02-02T16:30:33Z, size = 18215, hashes = { sha256 = "8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1" } }, + { url = "https://files.pythonhosted.org/packages/48/d6/e7cd795fc710292c3af3a06d80868ce4b02bfbbf370b7cee11d282815a2a/MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", upload-time = 2024-02-02T16:30:34Z, size = 14069, hashes = { sha256 = "3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4" } }, + { url = "https://files.pythonhosted.org/packages/51/b5/5d8ec796e2a08fc814a2c7d2584b55f889a55cf17dd1a90f2beb70744e5c/MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", upload-time = 2024-02-02T16:30:35Z, size = 29452, hashes = { sha256 = "ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee" } }, + { url = "https://files.pythonhosted.org/packages/0a/0d/2454f072fae3b5a137c119abf15465d1771319dfe9e4acbb31722a0fff91/MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", upload-time = 2024-02-02T16:30:36Z, size = 28462, hashes = { sha256 = "f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5" } }, + { url = "https://files.pythonhosted.org/packages/2d/75/fd6cb2e68780f72d47e6671840ca517bda5ef663d30ada7616b0462ad1e3/MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", upload-time = 2024-02-02T16:30:37Z, size = 27869, hashes = { sha256 = "ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b" } }, + { url = "https://files.pythonhosted.org/packages/b0/81/147c477391c2750e8fc7705829f7351cf1cd3be64406edcf900dc633feb2/MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", upload-time = 2024-02-02T16:30:39Z, size = 33906, hashes = { sha256 = "d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a" } }, + { url = "https://files.pythonhosted.org/packages/8b/ff/9a52b71839d7a256b563e85d11050e307121000dcebc97df120176b3ad93/MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", upload-time = 2024-02-02T16:30:40Z, size = 32296, hashes = { sha256 = "bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f" } }, + { url = "https://files.pythonhosted.org/packages/88/07/2dc76aa51b481eb96a4c3198894f38b480490e834479611a4053fbf08623/MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", upload-time = 2024-02-02T16:30:42Z, size = 33038, hashes = { sha256 = "58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169" } }, + { url = "https://files.pythonhosted.org/packages/96/0c/620c1fb3661858c0e37eb3cbffd8c6f732a67cd97296f725789679801b31/MarkupSafe-2.1.5-cp312-cp312-win32.whl", upload-time = 2024-02-02T16:30:43Z, size = 16572, hashes = { sha256 = "8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad" } }, + { url = "https://files.pythonhosted.org/packages/3f/14/c3554d512d5f9100a95e737502f4a2323a1959f6d0d01e0d0997b35f7b10/MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", upload-time = 2024-02-02T16:30:44Z, size = 17127, hashes = { sha256 = "823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb" } }, +] + +[[packages]] +name = "packaging" +version = "24.2" +sdist = { url = "https://files.pythonhosted.org/packages/d0/63/68dbb6eb2de9cb10ee4c9c14a0148804425e13c4fb20d61cce69f53106da/packaging-24.2.tar.gz", upload-time = 2024-11-08T09:47:47Z, size = 163950, hashes = { sha256 = "c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/88/ef/eb23f262cca3c0c4eb7ab1933c3b1f03d021f2c48f54763065b6f0e321be/packaging-24.2-py3-none-any.whl", upload-time = 2024-11-08T09:47:44Z, size = 65451, hashes = { sha256 = "09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759" } }] + +[[packages]] +name = "pygments" +version = "2.20.0" +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", upload-time = 2026-03-29T13:29:33Z, size = 4955991, hashes = { sha256 = "6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", upload-time = 2026-03-29T13:29:30Z, size = 1231151, hashes = { sha256 = "81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176" } }] + +[[packages]] +name = "python-docs-theme" +version = "2026.4" +sdist = { url = "https://files.pythonhosted.org/packages/fd/59/dbb07775a15ddf9f7f8d5f6ef4cd4da5e8afd908cc27e6585bb132e6366a/python_docs_theme-2026.4.tar.gz", upload-time = 2026-04-19T18:35:13Z, size = 29782, hashes = { sha256 = "a815f80c5a09f734449eb2498fbcbad05340976a7a543e431f57de92218a9315" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/db/05/b9298eb9330c70a3d1465a6116ab01dad095538c2e574a2d704bb0002f4d/python_docs_theme-2026.4-py3-none-any.whl", upload-time = 2026-04-19T18:35:12Z, size = 73742, hashes = { sha256 = "f755d80ebe8d7aa4fad8ee964ff999635c72eebd24ab10928a0e9726363d65fc" } }] + +[[packages]] +name = "requests" +version = "2.33.1" +sdist = { url = "https://files.pythonhosted.org/packages/5f/a4/98b9c7c6428a668bf7e42ebb7c79d576a1c3c1e3ae2d47e674b468388871/requests-2.33.1.tar.gz", upload-time = 2026-03-30T16:09:15Z, size = 134120, hashes = { sha256 = "18817f8c57c6263968bc123d237e3b8b08ac046f5456bd1e307ee8f4250d3517" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/d7/8e/7540e8a2036f79a125c1d2ebadf69ed7901608859186c856fa0388ef4197/requests-2.33.1-py3-none-any.whl", upload-time = 2026-03-30T16:09:13Z, size = 64947, hashes = { sha256 = "4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a" } }] + +[[packages]] +name = "roman-numerals" +version = "4.1.0" +sdist = { url = "https://files.pythonhosted.org/packages/ae/f9/41dc953bbeb056c17d5f7a519f50fdf010bd0553be2d630bc69d1e022703/roman_numerals-4.1.0.tar.gz", upload-time = 2025-12-17T18:25:34Z, size = 9077, hashes = { sha256 = "1af8b147eb1405d5839e78aeb93131690495fe9da5c91856cb33ad55a7f1e5b2" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl", upload-time = 2025-12-17T18:25:33Z, size = 7676, hashes = { sha256 = "647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7" } }] + +[[packages]] +name = "roman-numerals-py" +version = "4.1.0" +sdist = { url = "https://files.pythonhosted.org/packages/cb/b5/de96fca640f4f656eb79bbee0e79aeec52e3e0e359f8a3e6a0d366378b64/roman_numerals_py-4.1.0.tar.gz", upload-time = 2025-12-17T18:25:41Z, size = 4274, hashes = { sha256 = "f5d7b2b4ca52dd855ef7ab8eb3590f428c0b1ea480736ce32b01fef2a5f8daf9" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/27/2c/daca29684cbe9fd4bc711f8246da3c10adca1ccc4d24436b17572eb2590e/roman_numerals_py-4.1.0-py3-none-any.whl", upload-time = 2025-12-17T18:25:40Z, size = 4547, hashes = { sha256 = "553114c1167141c1283a51743759723ecd05604a1b6b507225e91dc1a6df0780" } }] + +[[packages]] +name = "snowballstemmer" +version = "2.2.0" +sdist = { url = "https://files.pythonhosted.org/packages/44/7b/af302bebf22c749c56c9c3e8ae13190b5b5db37a33d9068652e8f73b7089/snowballstemmer-2.2.0.tar.gz", upload-time = 2021-11-16T18:38:38Z, size = 86699, hashes = { sha256 = "09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/ed/dc/c02e01294f7265e63a7315fe086dd1df7dacb9f840a804da846b96d01b96/snowballstemmer-2.2.0-py2.py3-none-any.whl", upload-time = 2021-11-16T18:38:34Z, size = 93002, hashes = { sha256 = "c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a" } }] + +[[packages]] +name = "sphinx" +version = "8.2.3" +sdist = { url = "https://files.pythonhosted.org/packages/38/ad/4360e50ed56cb483667b8e6dadf2d3fda62359593faabbe749a27c4eaca6/sphinx-8.2.3.tar.gz", upload-time = 2025-03-02T22:31:59Z, size = 8321876, hashes = { sha256 = "398ad29dee7f63a75888314e9424d40f52ce5a6a87ae88e7071e80af296ec348" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/31/53/136e9eca6e0b9dc0e1962e2c908fbea2e5ac000c2a2fbd9a35797958c48b/sphinx-8.2.3-py3-none-any.whl", upload-time = 2025-03-02T22:31:56Z, size = 3589741, hashes = { sha256 = "4405915165f13521d875a8c29c8970800a0141c14cc5416a38feca4ea5d9b9c3" } }] + +[[packages]] +name = "sphinx-notfound-page" +version = "1.0.4" +sdist = { url = "https://files.pythonhosted.org/packages/73/7d/c545883c714319380325a52c9f80d093c97e718d812fd8090e42b1a08508/sphinx_notfound_page-1.0.4.tar.gz", upload-time = 2024-07-31T12:29:21Z, size = 519228, hashes = { sha256 = "2a52f49cd367b5c4e64072de1591cc367714098500abf4ecb9a3ecb4fec25aae" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/87/c4/877a5beffb8dcaf35e919c4c3cad56732c76370d106126394f4ca211ad7f/sphinx_notfound_page-1.0.4-py3-none-any.whl", upload-time = 2024-07-31T12:29:18Z, size = 8170, hashes = { sha256 = "f7c26ae0df3cf3d6f38f56b068762e6203d0ebb7e1c804de1059598d7dd8b9d8" } }] + +[[packages]] +name = "sphinxcontrib-applehelp" +version = "2.0.0" +sdist = { url = "https://files.pythonhosted.org/packages/ba/6e/b837e84a1a704953c62ef8776d45c3e8d759876b4a84fe14eba2859106fe/sphinxcontrib_applehelp-2.0.0.tar.gz", upload-time = 2024-07-29T01:09:00Z, size = 20053, hashes = { sha256 = "2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", upload-time = 2024-07-29T01:08:58Z, size = 119300, hashes = { sha256 = "4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5" } }] + +[[packages]] +name = "sphinxcontrib-devhelp" +version = "2.0.0" +sdist = { url = "https://files.pythonhosted.org/packages/f6/d2/5beee64d3e4e747f316bae86b55943f51e82bb86ecd325883ef65741e7da/sphinxcontrib_devhelp-2.0.0.tar.gz", upload-time = 2024-07-29T01:09:23Z, size = 12967, hashes = { sha256 = "411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", upload-time = 2024-07-29T01:09:21Z, size = 82530, hashes = { sha256 = "aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2" } }] + +[[packages]] +name = "sphinxcontrib-htmlhelp" +version = "2.1.0" +sdist = { url = "https://files.pythonhosted.org/packages/43/93/983afd9aa001e5201eab16b5a444ed5b9b0a7a010541e0ddfbbfd0b2470c/sphinxcontrib_htmlhelp-2.1.0.tar.gz", upload-time = 2024-07-29T01:09:37Z, size = 22617, hashes = { sha256 = "c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", upload-time = 2024-07-29T01:09:36Z, size = 98705, hashes = { sha256 = "166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8" } }] + +[[packages]] +name = "sphinxcontrib-jsmath" +version = "1.0.1" +sdist = { url = "https://files.pythonhosted.org/packages/b2/e8/9ed3830aeed71f17c026a07a5097edcf44b692850ef215b161b8ad875729/sphinxcontrib-jsmath-1.0.1.tar.gz", upload-time = 2019-01-21T16:10:16Z, size = 5787, hashes = { sha256 = "a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", upload-time = 2019-01-21T16:10:14Z, size = 5071, hashes = { sha256 = "2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178" } }] + +[[packages]] +name = "sphinxcontrib-qthelp" +version = "2.0.0" +sdist = { url = "https://files.pythonhosted.org/packages/68/bc/9104308fc285eb3e0b31b67688235db556cd5b0ef31d96f30e45f2e51cae/sphinxcontrib_qthelp-2.0.0.tar.gz", upload-time = 2024-07-29T01:09:56Z, size = 17165, hashes = { sha256 = "4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", upload-time = 2024-07-29T01:09:54Z, size = 88743, hashes = { sha256 = "b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb" } }] + +[[packages]] +name = "sphinxcontrib-serializinghtml" +version = "2.0.0" +sdist = { url = "https://files.pythonhosted.org/packages/3b/44/6716b257b0aa6bfd51a1b31665d1c205fb12cb5ad56de752dfa15657de2f/sphinxcontrib_serializinghtml-2.0.0.tar.gz", upload-time = 2024-07-29T01:10:09Z, size = 16080, hashes = { sha256 = "e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", upload-time = 2024-07-29T01:10:08Z, size = 92072, hashes = { sha256 = "6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331" } }] + +[[packages]] +name = "sphinxext-opengraph" +version = "0.13.0" +sdist = { url = "https://files.pythonhosted.org/packages/f6/c0/eb6838e3bae624ce6c8b90b245d17e84252863150e95efdb88f92c8aa3fb/sphinxext_opengraph-0.13.0.tar.gz", upload-time = 2025-08-29T12:20:31Z, size = 1026875, hashes = { sha256 = "103335d08567ad8468faf1425f575e3b698e9621f9323949a6c8b96d9793e80b" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/bf/a4/66c1fd4f8fab88faf71cee04a945f9806ba0fef753f2cfc8be6353f64508/sphinxext_opengraph-0.13.0-py3-none-any.whl", upload-time = 2025-08-29T12:20:29Z, size = 1004152, hashes = { sha256 = "936c07828edc9ad9a7b07908b29596dc84ed0b3ceaa77acdf51282d232d4d80e" } }] + +[[packages]] +name = "urllib3" +version = "2.6.3" +sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", upload-time = 2026-01-07T16:24:43Z, size = 435556, hashes = { sha256 = "1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", upload-time = 2026-01-07T16:24:42Z, size = 131584, hashes = { sha256 = "bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4" } }] diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 283b3b1bc6d..72e1cad3bbd 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -335,15 +335,29 @@ stored in the :mod:`sys` module is reset to its previous value:: :keyword:`!except*` clause -------------------------- -The :keyword:`!except*` clause(s) are used for handling -:exc:`ExceptionGroup`\s. The exception type for matching is interpreted as in -the case of :keyword:`except`, but in the case of exception groups we can have -partial matches when the type matches some of the exceptions in the group. -This means that multiple :keyword:`!except*` clauses can execute, -each handling part of the exception group. -Each clause executes at most once and handles an exception group -of all matching exceptions. Each exception in the group is handled by at most -one :keyword:`!except*` clause, the first that matches it. :: +The :keyword:`!except*` clause(s) specify one or more handlers for groups of +exceptions (:exc:`BaseExceptionGroup` instances). A :keyword:`try` statement +can have either :keyword:`except` or :keyword:`!except*` clauses, but not both. +The exception type for matching is mandatory in the case of :keyword:`!except*`, +so ``except*:`` is a syntax error. The type is interpreted as in the case of +:keyword:`!except`, but matching is performed on the exceptions contained in the +group that is being handled. An :exc:`TypeError` is raised if a matching +type is a subclass of :exc:`!BaseExceptionGroup`, because that would have +ambiguous semantics. + +When an exception group is raised in the try block, each :keyword:`!except*` +clause splits (see :meth:`~BaseExceptionGroup.split`) it into the subgroups +of matching and non-matching exceptions. If the matching subgroup is not empty, +it becomes the handled exception (the value returned from :func:`sys.exception`) +and assigned to the target of the :keyword:`!except*` clause (if there is one). +Then, the body of the :keyword:`!except*` clause executes. If the non-matching +subgroup is not empty, it is processed by the next :keyword:`!except*` in the +same manner. This continues until all exceptions in the group have been matched, +or the last :keyword:`!except*` clause has run. + +After all :keyword:`!except*` clauses execute, the group of unhandled exceptions +is merged with any exceptions that were raised or re-raised from within +:keyword:`!except*` clauses. This merged exception group propagates on.:: >>> try: ... raise ExceptionGroup("eg", @@ -356,37 +370,27 @@ one :keyword:`!except*` clause, the first that matches it. :: caught with nested (TypeError(2),) caught with nested (OSError(3), OSError(4)) + Exception Group Traceback (most recent call last): - | File "", line 2, in - | ExceptionGroup: eg + | File "", line 2, in + | raise ExceptionGroup("eg", + | [ValueError(1), TypeError(2), OSError(3), OSError(4)]) + | ExceptionGroup: eg (1 sub-exception) +-+---------------- 1 ---------------- | ValueError: 1 +------------------------------------ - -Any remaining exceptions that were not handled by any :keyword:`!except*` -clause are re-raised at the end, along with all exceptions that were -raised from within the :keyword:`!except*` clauses. If this list contains -more than one exception to reraise, they are combined into an exception -group. - -If the raised exception is not an exception group and its type matches -one of the :keyword:`!except*` clauses, it is caught and wrapped by an -exception group with an empty message string. :: +If the exception raised from the :keyword:`try` block is not an exception group +and its type matches one of the :keyword:`!except*` clauses, it is caught and +wrapped by an exception group with an empty message string. This ensures that the +type of the target ``e`` is consistently :exc:`BaseExceptionGroup`:: >>> try: ... raise BlockingIOError ... except* BlockingIOError as e: ... print(repr(e)) ... - ExceptionGroup('', (BlockingIOError())) + ExceptionGroup('', (BlockingIOError(),)) -An :keyword:`!except*` clause must have a matching expression; it cannot be ``except*:``. -Furthermore, this expression cannot contain exception group types, because that would -have ambiguous semantics. - -It is not possible to mix :keyword:`except` and :keyword:`!except*` -in the same :keyword:`try`. -The :keyword:`break`, :keyword:`continue`, and :keyword:`return` statements +:keyword:`break`, :keyword:`continue` and :keyword:`return` cannot appear in an :keyword:`!except*` clause. @@ -540,9 +544,9 @@ The following code:: is semantically equivalent to:: manager = (EXPRESSION) - enter = type(manager).__enter__ - exit = type(manager).__exit__ - value = enter(manager) + enter = manager.__enter__ + exit = manager.__exit__ + value = enter() hit_except = False try: @@ -550,11 +554,14 @@ is semantically equivalent to:: SUITE except: hit_except = True - if not exit(manager, *sys.exc_info()): + if not exit(*sys.exc_info()): raise finally: if not hit_except: - exit(manager, None, None, None) + exit(None, None, None) + +except that implicit :ref:`special method lookup ` is used +for :meth:`~object.__enter__` and :meth:`~object.__exit__`. With more than one item, the context managers are processed as if multiple :keyword:`with` statements were nested:: @@ -590,6 +597,7 @@ the items are surrounded by parentheses. For example:: statement. .. _match: +.. _case: The :keyword:`!match` statement =============================== @@ -850,7 +858,7 @@ A literal pattern corresponds to most : | "None" : | "True" : | "False" - signed_number: ["-"] NUMBER + signed_number: ["+" | "-"] NUMBER The rule ``strings`` and the token ``NUMBER`` are defined in the :doc:`standard Python grammar <./grammar>`. Triple-quoted strings are @@ -1674,13 +1682,12 @@ The following code:: Is semantically equivalent to:: - iter = (ITER) - iter = type(iter).__aiter__(iter) + iter = (ITER).__aiter__() running = True while running: try: - TARGET = await type(iter).__anext__(iter) + TARGET = await iter.__anext__() except StopAsyncIteration: running = False else: @@ -1688,7 +1695,8 @@ Is semantically equivalent to:: else: SUITE2 -See also :meth:`~object.__aiter__` and :meth:`~object.__anext__` for details. +except that implicit :ref:`special method lookup ` is used +for :meth:`~object.__aiter__` and :meth:`~object.__anext__`. It is a :exc:`SyntaxError` to use an ``async for`` statement outside the body of a coroutine function. @@ -1714,9 +1722,9 @@ The following code:: is semantically equivalent to:: manager = (EXPRESSION) - aenter = type(manager).__aenter__ - aexit = type(manager).__aexit__ - value = await aenter(manager) + aenter = manager.__aenter__ + aexit = manager.__aexit__ + value = await aenter() hit_except = False try: @@ -1724,13 +1732,14 @@ is semantically equivalent to:: SUITE except: hit_except = True - if not await aexit(manager, *sys.exc_info()): + if not await aexit(*sys.exc_info()): raise finally: if not hit_except: - await aexit(manager, None, None, None) + await aexit(None, None, None) -See also :meth:`~object.__aenter__` and :meth:`~object.__aexit__` for details. +except that implicit :ref:`special method lookup ` is used +for :meth:`~object.__aenter__` and :meth:`~object.__aexit__`. It is a :exc:`SyntaxError` to use an ``async with`` statement outside the body of a coroutine function. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 68645d1da3d..aef5bbe151c 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -16,9 +16,8 @@ Objects, values and types single: data :dfn:`Objects` are Python's abstraction for data. All data in a Python program -is represented by objects or by relations between objects. (In a sense, and in -conformance to Von Neumann's model of a "stored program computer", code is also -represented by objects.) +is represented by objects or by relations between objects. Even code is +represented by objects. .. index:: pair: built-in function; id @@ -29,9 +28,6 @@ represented by objects.) single: mutable object single: immutable object -.. XXX it *is* now possible in some cases to change an object's - type, under certain controlled conditions - Every object has an identity, a type and a value. An object's *identity* never changes once it has been created; you may think of it as the object's address in memory. The :keyword:`is` operator compares the identity of two objects; the @@ -294,6 +290,7 @@ floating-point numbers. The same caveats apply as for floating-point numbers. The real and imaginary parts of a complex number ``z`` can be retrieved through the read-only attributes ``z.real`` and ``z.imag``. +.. _datamodel-sequences: Sequences --------- @@ -313,12 +310,25 @@ including built-in sequences, interpret negative subscripts by adding the sequence length. For example, ``a[-2]`` equals ``a[n-2]``, the second to last item of sequence a with length ``n``. -.. index:: single: slicing +The resulting value must be a nonnegative integer less than the number of items +in the sequence. If it is not, an :exc:`IndexError` is raised. -Sequences also support slicing: ``a[i:j]`` selects all items with index *k* such -that *i* ``<=`` *k* ``<`` *j*. When used as an expression, a slice is a -sequence of the same type. The comment above about negative indexes also applies +.. index:: + single: slicing + single: start (slice object attribute) + single: stop (slice object attribute) + single: step (slice object attribute) + +Sequences also support slicing: ``a[start:stop]`` selects all items with index *k* such +that *start* ``<=`` *k* ``<`` *stop*. When used as an expression, a slice is a +sequence of the same type. The comment above about negative subscripts also applies to negative slice positions. +Note that no error is raised if a slice position is less than zero or larger +than the length of the sequence. + +If *start* is missing or :data:`None`, slicing behaves as if *start* was zero. +If *stop* is missing or ``None``, slicing behaves as if *stop* was equal to +the length of the sequence. Some sequences also support "extended slicing" with a third "step" parameter: ``a[i:j:k]`` selects all items of *a* with index *x* where ``x = i + n*k``, *n* @@ -349,17 +359,22 @@ Strings pair: built-in function; chr pair: built-in function; ord single: character - single: integer + pair: string; item single: Unicode - A string is a sequence of values that represent Unicode code points. - All the code points in the range ``U+0000 - U+10FFFF`` can be - represented in a string. Python doesn't have a :c:expr:`char` type; - instead, every code point in the string is represented as a string - object with length ``1``. The built-in function :func:`ord` + A string (:class:`str`) is a sequence of values that represent + :dfn:`characters`, or more formally, *Unicode code points*. + All the code points in the range ``0`` to ``0x10FFFF`` can be + represented in a string. + + Python doesn't have a dedicated *character* type. + Instead, every code point in the string is represented as a string + object with length ``1``. + + The built-in function :func:`ord` converts a code point from its string form to an integer in the - range ``0 - 10FFFF``; :func:`chr` converts an integer in the range - ``0 - 10FFFF`` to the corresponding length ``1`` string object. + range ``0`` to ``0x10FFFF``; :func:`chr` converts an integer in the range + ``0`` to ``0x10FFFF`` to the corresponding length ``1`` string object. :meth:`str.encode` can be used to convert a :class:`str` to :class:`bytes` using the given text encoding, and :meth:`bytes.decode` can be used to achieve the opposite. @@ -370,7 +385,7 @@ Tuples pair: singleton; tuple pair: empty; tuple - The items of a tuple are arbitrary Python objects. Tuples of two or + The items of a :class:`tuple` are arbitrary Python objects. Tuples of two or more items are formed by comma-separated lists of expressions. A tuple of one item (a 'singleton') can be formed by affixing a comma to an expression (an expression by itself does not create a tuple, since @@ -380,7 +395,7 @@ Tuples Bytes .. index:: bytes, byte - A bytes object is an immutable array. The items are 8-bit bytes, + A :class:`bytes` object is an immutable array. The items are 8-bit bytes, represented by integers in the range 0 <= x < 256. Bytes literals (like ``b'abc'``) and the built-in :func:`bytes` constructor can be used to create bytes objects. Also, bytes objects can be @@ -453,7 +468,7 @@ Sets These represent a mutable set. They are created by the built-in :func:`set` constructor and can be modified afterwards by several methods, such as - :meth:`add `. + :meth:`~set.add`. Frozen sets @@ -465,6 +480,8 @@ Frozen sets a dictionary key. +.. _datamodel-mappings: + Mappings -------- @@ -550,6 +567,7 @@ Special read-only attributes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. index:: + single: __builtins__ (function attribute) single: __closure__ (function attribute) single: __globals__ (function attribute) pair: global; namespace @@ -560,6 +578,12 @@ Special read-only attributes * - Attribute - Meaning + * - .. attribute:: function.__builtins__ + - A reference to the :class:`dictionary ` that holds the function's + builtins namespace. + + .. versionadded:: 3.10 + * - .. attribute:: function.__globals__ - A reference to the :class:`dictionary ` that holds the function's :ref:`global variables ` -- the global namespace of the module @@ -899,10 +923,10 @@ Attribute assignment updates the module's namespace dictionary, e.g., single: __loader__ (module attribute) single: __path__ (module attribute) single: __file__ (module attribute) - single: __cached__ (module attribute) single: __doc__ (module attribute) single: __annotations__ (module attribute) single: __annotate__ (module attribute) + single: __lazy_modules__ (module attribute) pair: module; namespace .. _import-mod-attrs: @@ -1048,43 +1072,28 @@ this approach. instead of :attr:`!module.__path__`. .. attribute:: module.__file__ -.. attribute:: module.__cached__ - :attr:`!__file__` and :attr:`!__cached__` are both optional attributes that + :attr:`!__file__` is an optional attribute that may or may not be set. Both attributes should be a :class:`str` when they are available. - :attr:`!__file__` indicates the pathname of the file from which the module - was loaded (if loaded from a file), or the pathname of the shared library - file for extension modules loaded dynamically from a shared library. - It might be missing for certain types of modules, such as C modules that are - statically linked into the interpreter, and the + An optional attribute, :attr:`!__file__` indicates the pathname of the file + from which the module was loaded (if loaded from a file), or the pathname of + the shared library file for extension modules loaded dynamically from a + shared library. It might be missing for certain types of modules, such as C + modules that are statically linked into the interpreter, and the :ref:`import system ` may opt to leave it unset if it has no semantic meaning (for example, a module loaded from a database). - If :attr:`!__file__` is set then the :attr:`!__cached__` attribute might - also be set, which is the path to any compiled version of - the code (for example, a byte-compiled file). The file does not need to exist - to set this attribute; the path can simply point to where the - compiled file *would* exist (see :pep:`3147`). - - Note that :attr:`!__cached__` may be set even if :attr:`!__file__` is not - set. However, that scenario is quite atypical. Ultimately, the - :term:`loader` is what makes use of the module spec provided by the - :term:`finder` (from which :attr:`!__file__` and :attr:`!__cached__` are - derived). So if a loader can load from a cached module but otherwise does - not load from a file, that atypical scenario may be appropriate. - - It is **strongly** recommended that you use - :attr:`module.__spec__.cached ` - instead of :attr:`!module.__cached__`. - .. deprecated-removed:: 3.13 3.15 - Setting :attr:`!__cached__` on a module while failing to set + Setting ``__cached__`` on a module while failing to set :attr:`!__spec__.cached` is deprecated. In Python 3.15, - :attr:`!__cached__` will cease to be set or taken into consideration by + ``__cached__`` will cease to be set or taken into consideration by the import system or standard library. + .. versionchanged:: 3.15 + ``__cached__`` is no longer set. + Other writable attributes on module objects ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1113,6 +1122,20 @@ the following writable attributes: .. versionadded:: 3.14 +.. attribute:: module.__lazy_modules__ + + A container (an object implementing :meth:`~object.__contains__`) of fully + qualified module name strings. When defined + at module scope, any regular :keyword:`import` statement in that module whose + target module name appears in this container is treated as a + :ref:`lazy import `, as if the :keyword:`lazy` keyword had + been used. Imports inside functions, class bodies, or + :keyword:`try`/:keyword:`except`/:keyword:`finally` blocks are unaffected. + + See :ref:`lazy-modules-compat` for details and examples. + + .. versionadded:: 3.15 + Module dictionaries ^^^^^^^^^^^^^^^^^^^ @@ -1185,6 +1208,7 @@ Special attributes single: __module__ (class attribute) single: __dict__ (class attribute) single: __bases__ (class attribute) + single: __base__ (class attribute) single: __doc__ (class attribute) single: __annotations__ (class attribute) single: __annotate__ (class attribute) @@ -1219,6 +1243,13 @@ Special attributes In most cases, for a class defined as ``class X(A, B, C)``, ``X.__bases__`` will be exactly equal to ``(A, B, C)``. + * - .. attribute:: type.__base__ + - .. impl-detail:: + + The single base class in the inheritance chain that is responsible + for the memory layout of instances. This attribute corresponds to + :c:member:`~PyTypeObject.tp_base` at the C level. + * - .. attribute:: type.__doc__ - The class's documentation string, or ``None`` if undefined. Not inherited by subclasses. @@ -1385,12 +1416,28 @@ also :func:`os.popen`, :func:`os.fdopen`, and the :meth:`~socket.socket.makefile` method of socket objects (and perhaps by other functions or methods provided by extension modules). +File objects implement common methods, listed below, to simplify usage in +generic code. They are expected to be :ref:`context-managers`. + The objects ``sys.stdin``, ``sys.stdout`` and ``sys.stderr`` are initialized to file objects corresponding to the interpreter's standard input, output and error streams; they are all open in text mode and therefore follow the interface defined by the :class:`io.TextIOBase` abstract class. +.. method:: file.read(size=-1, /) + + Retrieve up to *size* data from the file. As a convenience if *size* is + unspecified or -1 retrieve all data available. + +.. method:: file.write(data, /) + + Store *data* to the file. + +.. method:: file.close() + + Flush any buffers and close the underlying file. + Internal types -------------- @@ -1429,7 +1476,6 @@ indirectly) to mutable objects. single: co_filename (code object attribute) single: co_firstlineno (code object attribute) single: co_flags (code object attribute) - single: co_lnotab (code object attribute) single: co_name (code object attribute) single: co_names (code object attribute) single: co_nlocals (code object attribute) @@ -1502,14 +1548,6 @@ Special read-only attributes * - .. attribute:: codeobject.co_firstlineno - The line number of the first line of the function - * - .. attribute:: codeobject.co_lnotab - - A string encoding the mapping from :term:`bytecode` offsets to line - numbers. For details, see the source code of the interpreter. - - .. deprecated:: 3.12 - This attribute of code objects is deprecated, and may be removed in - Python 3.15. - * - .. attribute:: codeobject.co_stacksize - The required stack size of the code object @@ -1831,6 +1869,12 @@ Slice objects are used to represent slices for :meth:`~object.__getitem__` methods. They are also created by the built-in :func:`slice` function. +.. versionadded:: 3.15 + + The :func:`slice` type now supports :ref:`subscription `. For + example, ``slice[float]`` may be used in type annotations to indicate a slice + containing :type:`float` objects. + .. index:: single: start (slice object attribute) single: stop (slice object attribute) @@ -2234,7 +2278,7 @@ Basic customization This is intended to provide protection against a denial-of-service caused by carefully chosen inputs that exploit the worst case performance of a dict insertion, *O*\ (*n*\ :sup:`2`) complexity. See - http://ocert.org/advisories/ocert-2011-003.html for details. + https://ocert.org/advisories/ocert-2011-003.html for details. Changing hash values affects the iteration order of sets. Python has never made guarantees about this ordering @@ -2558,7 +2602,7 @@ instance dictionary. In contrast, non-data descriptors can be overridden by instances. Python methods (including those decorated with -:func:`@staticmethod ` and :func:`@classmethod `) are +:deco:`staticmethod` and :deco:`classmethod`) are implemented as non-data descriptors. Accordingly, instances can redefine and override methods. This allows individual instances to acquire behaviors that differ from other instances of the same class. @@ -2626,10 +2670,10 @@ Notes on using *__slots__*: descriptor directly from the base class). This renders the meaning of the program undefined. In the future, a check may be added to prevent this. -* :exc:`TypeError` will be raised if nonempty *__slots__* are defined for a - class derived from a +* :exc:`TypeError` will be raised if *__slots__* other than *__dict__* and + *__weakref__* are defined for a class derived from a :c:member:`"variable-length" built-in type ` such as - :class:`int`, :class:`bytes`, and :class:`tuple`. + :class:`int`, :class:`bytes`, and :class:`type`, except :class:`tuple`. * Any non-string :term:`iterable` may be assigned to *__slots__*. @@ -2652,6 +2696,11 @@ Notes on using *__slots__*: of the iterator's values. However, the *__slots__* attribute will be an empty iterator. +.. versionchanged:: 3.15 + Allowed defining the *__dict__* and *__weakref__* *__slots__* for any class. + Allowed defining any *__slots__* for a class derived from :class:`tuple`. + + .. _class-customization: Customizing class creation @@ -2993,7 +3042,7 @@ class method ``__class_getitem__()``. When defined on a class, ``__class_getitem__()`` is automatically a class method. As such, there is no need for it to be decorated with - :func:`@classmethod` when it is defined. + :deco:`classmethod` when it is defined. The purpose of *__class_getitem__* @@ -3142,17 +3191,20 @@ objects. The :mod:`collections.abc` module provides a :term:`abstract base class` to help create those methods from a base set of :meth:`~object.__getitem__`, :meth:`~object.__setitem__`, :meth:`~object.__delitem__`, and :meth:`!keys`. -Mutable sequences should provide methods :meth:`~sequence.append`, -:meth:`~sequence.count`, :meth:`~sequence.index`, :meth:`~sequence.extend`, -:meth:`~sequence.insert`, :meth:`~sequence.pop`, :meth:`~sequence.remove`, -:meth:`~sequence.reverse` and :meth:`!sort`, + +Mutable sequences should provide methods +:meth:`~sequence.append`, :meth:`~sequence.clear`, :meth:`~sequence.count`, +:meth:`~sequence.extend`, :meth:`~sequence.index`, :meth:`~sequence.insert`, +:meth:`~sequence.pop`, :meth:`~sequence.remove`, and :meth:`~sequence.reverse`, like Python standard :class:`list` objects. Finally, sequence types should implement addition (meaning concatenation) and multiplication (meaning repetition) by defining the methods :meth:`~object.__add__`, :meth:`~object.__radd__`, :meth:`~object.__iadd__`, :meth:`~object.__mul__`, :meth:`~object.__rmul__` and :meth:`~object.__imul__` described below; they should not define other numerical -operators. It is recommended that both mappings and sequences implement the +operators. + +It is recommended that both mappings and sequences implement the :meth:`~object.__contains__` method to allow efficient use of the ``in`` operator; for mappings, ``in`` should search the mapping's keys; for sequences, it should @@ -3193,43 +3245,55 @@ through the object's keys; for sequences, it should iterate through the values. .. versionadded:: 3.4 -.. index:: pair: object; slice +.. method:: object.__getitem__(self, subscript) -.. note:: + Called to implement *subscription*, that is, ``self[subscript]``. + See :ref:`subscriptions` for details on the syntax. - Slicing is done exclusively with the following three methods. A call like :: + There are two types of built-in objects that support subscription + via :meth:`!__getitem__`: - a[1:2] = b + - **sequences**, where *subscript* (also called + :term:`index`) should be an integer or a :class:`slice` object. + See the :ref:`sequence documentation ` for the expected + behavior, including handling :class:`slice` objects and negative indices. + - **mappings**, where *subscript* is also called the :term:`key`. + See :ref:`mapping documentation ` for the expected + behavior. - is translated to :: + If *subscript* is of an inappropriate type, :meth:`!__getitem__` + should raise :exc:`TypeError`. + If *subscript* has an inappropriate value, :meth:`!__getitem__` + should raise an :exc:`LookupError` or one of its subclasses + (:exc:`IndexError` for sequences; :exc:`KeyError` for mappings). - a[slice(1, 2, None)] = b - - and so forth. Missing slice items are always filled in with ``None``. - - -.. method:: object.__getitem__(self, key) - - Called to implement evaluation of ``self[key]``. For :term:`sequence` types, - the accepted keys should be integers. Optionally, they may support - :class:`slice` objects as well. Negative index support is also optional. - If *key* is - of an inappropriate type, :exc:`TypeError` may be raised; if *key* is a value - outside the set of indexes for the sequence (after any special - interpretation of negative values), :exc:`IndexError` should be raised. For - :term:`mapping` types, if *key* is missing (not in the container), - :exc:`KeyError` should be raised. + .. index:: pair: object; slice .. note:: - :keyword:`for` loops expect that an :exc:`IndexError` will be raised for - illegal indexes to allow proper detection of the end of the sequence. + Slicing is handled by :meth:`!__getitem__`, :meth:`~object.__setitem__`, + and :meth:`~object.__delitem__`. + A call like :: + + a[1:2] = b + + is translated to :: + + a[slice(1, 2, None)] = b + + and so forth. Missing slice items are always filled in with ``None``. .. note:: - When :ref:`subscripting` a *class*, the special + The sequence iteration protocol (used, for example, in :keyword:`for` + loops), expects that an :exc:`IndexError` will be raised for illegal + indexes to allow proper detection of the end of a sequence. + + .. note:: + + When :ref:`subscripting ` a *class*, the special class method :meth:`~object.__class_getitem__` may be called instead of - ``__getitem__()``. See :ref:`classgetitem-versus-getitem` for more + :meth:`!__getitem__`. See :ref:`classgetitem-versus-getitem` for more details. @@ -3579,12 +3643,25 @@ implement the protocol in Python. provides a convenient way to interpret the flags. The method must return a :class:`memoryview` object. + **Thread safety:** In :term:`free-threaded ` Python, + implementations must manage any internal export counter using atomic + operations. The method must be safe to call concurrently from multiple + threads, and the returned buffer's underlying data must remain valid + until the corresponding :meth:`~object.__release_buffer__` call + completes. See :ref:`thread-safety-memoryview` for details. + .. method:: object.__release_buffer__(self, buffer) Called when a buffer is no longer needed. The *buffer* argument is a :class:`memoryview` object that was previously returned by :meth:`~object.__buffer__`. The method must release any resources associated with the buffer. This method should return ``None``. + + **Thread safety:** In :term:`free-threaded ` Python, + any export counter decrement must use atomic operations. Resource + cleanup must be thread-safe, as the final release may race with + concurrent releases from other threads. + Buffer objects that do not need to perform any cleanup are not required to implement this method. diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index cb6c524dd97..639c232571e 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -398,6 +398,192 @@ See also the description of the :keyword:`try` statement in section :ref:`try` and :keyword:`raise` statement in section :ref:`raise`. +.. _execcomponents: + +Runtime Components +================== + +General Computing Model +----------------------- + +Python's execution model does not operate in a vacuum. It runs on +a host machine and through that host's runtime environment, including +its operating system (OS), if there is one. When a program runs, +the conceptual layers of how it runs on the host look something +like this: + + | **host machine** + | **process** (global resources) + | **thread** (runs machine code) + +Each process represents a program running on the host. Think of each +process itself as the data part of its program. Think of the process' +threads as the execution part of the program. This distinction will +be important to understand the conceptual Python runtime. + +The process, as the data part, is the execution context in which the +program runs. It mostly consists of the set of resources assigned to +the program by the host, including memory, signals, file handles, +sockets, and environment variables. + +Processes are isolated and independent from one another. (The same +is true for hosts.) The host manages the process' access to its +assigned resources, in addition to coordinating between processes. + +Each thread represents the actual execution of the program's machine +code, running relative to the resources assigned to the program's +process. It's strictly up to the host how and when that execution +takes place. + +From the point of view of Python, a program always starts with exactly +one thread. However, the program may grow to run in multiple +simultaneous threads. Not all hosts support multiple threads per +process, but most do. Unlike processes, threads in a process are not +isolated and independent from one another. Specifically, all threads +in a process share all of the process' resources. + +The fundamental point of threads is that each one does *run* +independently, at the same time as the others. That may be only +conceptually at the same time ("concurrently") or physically +("in parallel"). Either way, the threads effectively run +at a non-synchronized rate. + +.. note:: + + That non-synchronized rate means none of the process' memory is + guaranteed to stay consistent for the code running in any given + thread. Thus multi-threaded programs must take care to coordinate + access to intentionally shared resources. Likewise, they must take + care to be absolutely diligent about not accessing any *other* + resources in multiple threads; otherwise two threads running at the + same time might accidentally interfere with each other's use of some + shared data. All this is true for both Python programs and the + Python runtime. + + The cost of this broad, unstructured requirement is the tradeoff for + the kind of raw concurrency that threads provide. The alternative + to the required discipline generally means dealing with + non-deterministic bugs and data corruption. + +Python Runtime Model +-------------------- + +The same conceptual layers apply to each Python program, with some +extra data layers specific to Python: + + | **host machine** + | **process** (global resources) + | Python global runtime (*state*) + | Python interpreter (*state*) + | **thread** (runs Python bytecode and "C-API") + | Python thread *state* + +At the conceptual level: when a Python program starts, it looks exactly +like that diagram, with one of each. The runtime may grow to include +multiple interpreters, and each interpreter may grow to include +multiple thread states. + +.. note:: + + A Python implementation won't necessarily implement the runtime + layers distinctly or even concretely. The only exception is places + where distinct layers are directly specified or exposed to users, + like through the :mod:`threading` module. + +.. note:: + + The initial interpreter is typically called the "main" interpreter. + Some Python implementations, like CPython, assign special roles + to the main interpreter. + + Likewise, the host thread where the runtime was initialized is known + as the "main" thread. It may be different from the process' initial + thread, though they are often the same. In some cases "main thread" + may be even more specific and refer to the initial thread state. + A Python runtime might assign specific responsibilities + to the main thread, such as handling signals. + +As a whole, the Python runtime consists of the global runtime state, +interpreters, and thread states. The runtime ensures all that state +stays consistent over its lifetime, particularly when used with +multiple host threads. + +The global runtime, at the conceptual level, is just a set of +interpreters. While those interpreters are otherwise isolated and +independent from one another, they may share some data or other +resources. The runtime is responsible for managing these global +resources safely. The actual nature and management of these resources +is implementation-specific. Ultimately, the external utility of the +global runtime is limited to managing interpreters. + +In contrast, an "interpreter" is conceptually what we would normally +think of as the (full-featured) "Python runtime". When machine code +executing in a host thread interacts with the Python runtime, it calls +into Python in the context of a specific interpreter. + +.. note:: + + The term "interpreter" here is not the same as the "bytecode + interpreter", which is what regularly runs in threads, executing + compiled Python code. + + In an ideal world, "Python runtime" would refer to what we currently + call "interpreter". However, it's been called "interpreter" at least + since introduced in 1997 (`CPython:a027efa5b`_). + + .. _CPython:a027efa5b: https://github.com/python/cpython/commit/a027efa5b + +Each interpreter completely encapsulates all of the non-process-global, +non-thread-specific state needed for the Python runtime to work. +Notably, the interpreter's state persists between uses. It includes +fundamental data like :data:`sys.modules`. The runtime ensures +multiple threads using the same interpreter will safely +share it between them. + +A Python implementation may support using multiple interpreters at the +same time in the same process. They are independent and isolated from +one another. For example, each interpreter has its own +:data:`sys.modules`. + +For thread-specific runtime state, each interpreter has a set of thread +states, which it manages, in the same way the global runtime contains +a set of interpreters. It can have thread states for as many host +threads as it needs. It may even have multiple thread states for +the same host thread, though that isn't as common. + +Each thread state, conceptually, has all the thread-specific runtime +data an interpreter needs to operate in one host thread. The thread +state includes the current raised exception and the thread's Python +call stack. It may include other thread-specific resources. + +.. note:: + + The term "Python thread" can sometimes refer to a thread state, but + normally it means a thread created using the :mod:`threading` module. + +Each thread state, over its lifetime, is always tied to exactly one +interpreter and exactly one host thread. It will only ever be used in +that thread and with that interpreter. + +Multiple thread states may be tied to the same host thread, whether for +different interpreters or even the same interpreter. However, for any +given host thread, only one of the thread states tied to it can be used +by the thread at a time. + +Thread states are isolated and independent from one another and don't +share any data, except for possibly sharing an interpreter and objects +or other resources belonging to that interpreter. + +Once a program is running, new Python threads can be created using the +:mod:`threading` module (on platforms and Python implementations that +support threads). Additional processes can be created using the +:mod:`os`, :mod:`subprocess`, and :mod:`multiprocessing` modules. +Interpreters can be created and used with the +:mod:`~concurrent.interpreters` module. Coroutines (async) can +be run using :mod:`asyncio` in each interpreter, typically only +in a single thread (often the main thread). + + .. rubric:: Footnotes .. [#] This limitation occurs because the code that is executed by these operations diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 9aca25e3214..68dcfc00bbd 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -9,9 +9,11 @@ Expressions This chapter explains the meaning of the elements of expressions in Python. -**Syntax Notes:** In this and the following chapters, extended BNF notation will -be used to describe syntax, not lexical analysis. When (one alternative of) a -syntax rule has the form +**Syntax Notes:** In this and the following chapters, +:ref:`grammar notation ` will be used to describe syntax, +not lexical analysis. + +When (one alternative of) a syntax rule has the form: .. productionlist:: python-grammar name: othername @@ -29,17 +31,13 @@ Arithmetic conversions When a description of an arithmetic operator below uses the phrase "the numeric arguments are converted to a common real type", this means that the operator -implementation for built-in types works as follows: +implementation for built-in numeric types works as described in the +:ref:`Numeric Types ` section of the standard +library documentation. -* If both arguments are complex numbers, no conversion is performed; - -* if either argument is a complex or a floating-point number, the other is converted to a floating-point number; - -* otherwise, both must be integers and no conversion is necessary. - -Some additional rules apply for certain operators (e.g., a string as a left -argument to the '%' operator). Extensions must define their own conversion -behavior. +Some additional rules apply for certain operators and non-numeric operands +(for example, a string as a left argument to the ``%`` operator). +Extensions must define their own conversion behavior. .. _atoms: @@ -49,15 +47,57 @@ Atoms .. index:: atom -Atoms are the most basic elements of expressions. The simplest atoms are -identifiers or literals. Forms enclosed in parentheses, brackets or braces are -also categorized syntactically as atoms. The syntax for atoms is: +Atoms are the most basic elements of expressions. +The simplest atoms are :ref:`names ` or literals. +Forms enclosed in parentheses, brackets or braces are also categorized +syntactically as atoms. -.. productionlist:: python-grammar - atom: `identifier` | `literal` | `enclosure` - enclosure: `parenth_form` | `list_display` | `dict_display` | `set_display` - : | `generator_expression` | `yield_atom` +Formally, the syntax for atoms is: +.. grammar-snippet:: + :group: python-grammar + + atom: + | 'True' + | 'False' + | 'None' + | '...' + | `identifier` + | `literal` + | `enclosure` + enclosure: + | `parenth_form` + | `list_display` + | `dict_display` + | `set_display` + | `generator_expression` + | `yield_atom` + + +.. _atom-singletons: + +Built-in constants +------------------ + +The keywords ``True``, ``False``, and ``None`` name +:ref:`built-in constants `. +The token ``...`` names the :py:data:`Ellipsis` constant. + +Evaluation of these atoms yields the corresponding value. + +.. note:: + + Several more built-in constants are available as global variables, + but only the ones mentioned here are :ref:`keywords `. + In particular, these names cannot be reassigned or used as attributes: + + .. code-block:: pycon + + >>> False = 123 + File "", line 1 + False = 123 + ^^^^^ + SyntaxError: cannot assign to False .. _atom-identifiers: @@ -131,51 +171,104 @@ Literals .. index:: single: literal -Python supports string and bytes literals and various numeric literals: +A :dfn:`literal` is a textual representation of a value. +Python supports numeric, string and bytes literals. +:ref:`Format strings ` and :ref:`template strings ` +are treated as string literals. + +Numeric literals consist of a single :token:`NUMBER ` +token, which names an integer, floating-point number, or an imaginary number. +See the :ref:`numbers` section in Lexical analysis documentation for details. + +String and bytes literals may consist of several tokens. +See section :ref:`string-concatenation` for details. + +Note that negative and complex numbers, like ``-3`` or ``3+4.2j``, +are syntactically not literals, but :ref:`unary ` or +:ref:`binary ` arithmetic operations involving the ``-`` or ``+`` +operator. + +Evaluation of a literal yields an object of the given type +(:class:`int`, :class:`float`, :class:`complex`, :class:`str`, +:class:`bytes`, or :class:`~string.templatelib.Template`) with the given value. +The value may be approximated in the case of floating-point +and imaginary literals. + +The formal grammar for literals is: .. grammar-snippet:: :group: python-grammar literal: `strings` | `NUMBER` -Evaluation of a literal yields an object of the given type (string, bytes, -integer, floating-point number, complex number) with the given value. The value -may be approximated in the case of floating-point and imaginary (complex) -literals. -See section :ref:`literals` for details. -See section :ref:`string-concatenation` for details on ``strings``. - .. index:: triple: immutable; data; type pair: immutable; object +Literals and object identity +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + All literals correspond to immutable data types, and hence the object's identity is less important than its value. Multiple evaluations of literals with the same value (either the same occurrence in the program text or a different occurrence) may obtain the same object or a different object with the same value. +.. admonition:: CPython implementation detail + + For example, in CPython, *small* integers with the same value evaluate + to the same object:: + + >>> x = 7 + >>> y = 7 + >>> x is y + True + + However, large integers evaluate to different objects:: + + >>> x = 123456789 + >>> y = 123456789 + >>> x is y + False + + This behavior may change in future versions of CPython. + In particular, the boundary between "small" and "large" integers has + already changed in the past. + + CPython will emit a :py:exc:`SyntaxWarning` when you compare literals + using ``is``:: + + >>> x = 7 + >>> x is 7 + :1: SyntaxWarning: "is" with 'int' literal. Did you mean "=="? + True + + See :ref:`faq-identity-with-is` for more information. + +:ref:`Template strings ` are immutable but may reference mutable +objects as :class:`~string.templatelib.Interpolation` values. +For the purposes of this section, two t-strings have the "same value" if +both their structure and the *identity* of the values match. + +.. impl-detail:: + + Currently, each evaluation of a template string results in + a different object. + .. _string-concatenation: String literal concatenation ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Multiple adjacent string or bytes literals (delimited by whitespace), possibly +Multiple adjacent string or bytes literals, possibly using different quoting conventions, are allowed, and their meaning is the same as their concatenation:: >>> "hello" 'world' "helloworld" -Formally: - -.. grammar-snippet:: - :group: python-grammar - - strings: ( `STRING` | fstring)+ | tstring+ - This feature is defined at the syntactical level, so it only works with literals. To concatenate string expressions at run time, the '+' operator may be used:: @@ -208,6 +301,13 @@ string literals:: >>> t"Hello" t"{name}!" Template(strings=('Hello', '!'), interpolations=(...)) +Formally: + +.. grammar-snippet:: + :group: python-grammar + + strings: (`STRING` | `fstring`)+ | `tstring`+ + .. _parenthesized: @@ -266,17 +366,19 @@ called "displays", each of them in two flavors: Common syntax elements for comprehensions are: .. productionlist:: python-grammar - comprehension: `assignment_expression` `comp_for` + comprehension: `flexible_expression` `comp_for` comp_for: ["async"] "for" `target_list` "in" `or_test` [`comp_iter`] comp_iter: `comp_for` | `comp_if` comp_if: "if" `or_test` [`comp_iter`] The comprehension consists of a single expression followed by at least one -:keyword:`!for` clause and zero or more :keyword:`!for` or :keyword:`!if` clauses. -In this case, the elements of the new container are those that would be produced -by considering each of the :keyword:`!for` or :keyword:`!if` clauses a block, -nesting from left to right, and evaluating the expression to produce an element -each time the innermost block is reached. +:keyword:`!for` clause and zero or more :keyword:`!for` or :keyword:`!if` +clauses. In this case, the elements of the new container are those that would +be produced by considering each of the :keyword:`!for` or :keyword:`!if` +clauses a block, nesting from left to right, and evaluating the expression to +produce an element each time the innermost block is reached. If the expression +is starred, the result will instead be unpacked to produce zero or more +elements. However, aside from the iterable expression in the leftmost :keyword:`!for` clause, the comprehension is executed in a separate implicitly nested scope. This ensures @@ -321,6 +423,9 @@ See also :pep:`530`. asynchronous functions. Outer comprehensions implicitly become asynchronous. +.. versionchanged:: 3.15 + Unpacking with the ``*`` operator is now allowed in the expression. + .. _lists: @@ -396,8 +501,8 @@ enclosed in curly braces: .. productionlist:: python-grammar dict_display: "{" [`dict_item_list` | `dict_comprehension`] "}" dict_item_list: `dict_item` ("," `dict_item`)* [","] + dict_comprehension: `dict_item` `comp_for` dict_item: `expression` ":" `expression` | "**" `or_expr` - dict_comprehension: `expression` ":" `expression` `comp_for` A dictionary display yields a new dictionary object. @@ -419,10 +524,21 @@ earlier dict items and earlier dictionary unpackings. .. versionadded:: 3.5 Unpacking into dictionary displays, originally proposed by :pep:`448`. -A dict comprehension, in contrast to list and set comprehensions, needs two -expressions separated with a colon followed by the usual "for" and "if" clauses. -When the comprehension is run, the resulting key and value elements are inserted -in the new dictionary in the order they are produced. +A dict comprehension may take one of two forms: + +- The first form uses two expressions separated with a colon followed by the + usual "for" and "if" clauses. When the comprehension is run, the resulting + key and value elements are inserted in the new dictionary in the order they + are produced. + +- The second form uses a single expression prefixed by the ``**`` dictionary + unpacking operator followed by the usual "for" and "if" clauses. When the + comprehension is evaluated, the expression is evaluated and then unpacked, + inserting zero or more key/value pairs into the new dictionary. + +Both forms of dictionary comprehension retain the property that if the same key +is specified multiple times, the associated value in the resulting dictionary +will be the last one specified. .. index:: pair: immutable; object hashable @@ -439,6 +555,8 @@ prevails. the key. Starting with 3.8, the key is evaluated before the value, as proposed by :pep:`572`. +.. versionchanged:: 3.15 + Unpacking with the ``**`` operator is now allowed in dictionary comprehensions. .. _genexpr: @@ -453,7 +571,7 @@ Generator expressions A generator expression is a compact generator notation in parentheses: .. productionlist:: python-grammar - generator_expression: "(" `expression` `comp_for` ")" + generator_expression: "(" `flexible_expression` `comp_for` ")" A generator expression yields a new generator object. Its syntax is the same as for comprehensions, except that it is enclosed in parentheses instead of @@ -893,7 +1011,7 @@ Primaries represent the most tightly bound operations of the language. Their syntax is: .. productionlist:: python-grammar - primary: `atom` | `attributeref` | `subscription` | `slicing` | `call` + primary: `atom` | `attributeref` | `subscription` | `call` .. _attribute-references: @@ -932,8 +1050,8 @@ method, that method is called as a fallback. .. _subscriptions: -Subscriptions -------------- +Subscriptions and slicings +-------------------------- .. index:: single: subscription @@ -948,67 +1066,74 @@ Subscriptions pair: object; dictionary pair: sequence; item -The subscription of an instance of a :ref:`container class ` -will generally select an element from the container. The subscription of a -:term:`generic class ` will generally return a -:ref:`GenericAlias ` object. +The :dfn:`subscription` syntax is usually used for selecting an element from a +:ref:`container ` -- for example, to get a value from +a :class:`dict`:: -.. productionlist:: python-grammar - subscription: `primary` "[" `flexible_expression_list` "]" + >>> digits_by_name = {'one': 1, 'two': 2} + >>> digits_by_name['two'] # Subscripting a dictionary using the key 'two' + 2 -When an object is subscripted, the interpreter will evaluate the primary and -the expression list. +In the subscription syntax, the object being subscribed -- a +:ref:`primary ` -- is followed by a :dfn:`subscript` in +square brackets. +In the simplest case, the subscript is a single expression. -The primary must evaluate to an object that supports subscription. An object -may support subscription through defining one or both of -:meth:`~object.__getitem__` and :meth:`~object.__class_getitem__`. When the -primary is subscripted, the evaluated result of the expression list will be -passed to one of these methods. For more details on when ``__class_getitem__`` -is called instead of ``__getitem__``, see :ref:`classgetitem-versus-getitem`. +Depending on the type of the object being subscribed, the subscript is +sometimes called a :term:`key` (for mappings), :term:`index` (for sequences), +or *type argument* (for :term:`generic types `). +Syntactically, these are all equivalent:: -If the expression list contains at least one comma, or if any of the expressions -are starred, the expression list will evaluate to a :class:`tuple` containing -the items of the expression list. Otherwise, the expression list will evaluate -to the value of the list's sole member. + >>> colors = ['red', 'blue', 'green', 'black'] + >>> colors[3] # Subscripting a list using the index 3 + 'black' -.. versionchanged:: 3.11 - Expressions in an expression list may be starred. See :pep:`646`. + >>> list[str] # Parameterizing the list type using the type argument str + list[str] -For built-in objects, there are two types of objects that support subscription -via :meth:`~object.__getitem__`: +At runtime, the interpreter will evaluate the primary and +the subscript, and call the primary's :meth:`~object.__getitem__` or +:meth:`~object.__class_getitem__` :term:`special method` with the subscript +as argument. +For more details on which of these methods is called, see +:ref:`classgetitem-versus-getitem`. -1. Mappings. If the primary is a :term:`mapping`, the expression list must - evaluate to an object whose value is one of the keys of the mapping, and the - subscription selects the value in the mapping that corresponds to that key. - An example of a builtin mapping class is the :class:`dict` class. -2. Sequences. If the primary is a :term:`sequence`, the expression list must - evaluate to an :class:`int` or a :class:`slice` (as discussed in the - following section). Examples of builtin sequence classes include the - :class:`str`, :class:`list` and :class:`tuple` classes. +To show how subscription works, we can define a custom object that +implements :meth:`~object.__getitem__` and prints out the value of +the subscript:: -The formal syntax makes no special provision for negative indices in -:term:`sequences `. However, built-in sequences all provide a :meth:`~object.__getitem__` -method that interprets negative indices by adding the length of the sequence -to the index so that, for example, ``x[-1]`` selects the last item of ``x``. The -resulting value must be a nonnegative integer less than the number of items in -the sequence, and the subscription selects the item whose index is that value -(counting from zero). Since the support for negative indices and slicing -occurs in the object's :meth:`~object.__getitem__` method, subclasses overriding -this method will need to explicitly add that support. + >>> class SubscriptionDemo: + ... def __getitem__(self, key): + ... print(f'subscripted with: {key!r}') + ... + >>> demo = SubscriptionDemo() + >>> demo[1] + subscripted with: 1 + >>> demo['a' * 3] + subscripted with: 'aaa' -.. index:: - single: character - pair: string; item +See :meth:`~object.__getitem__` documentation for how built-in types handle +subscription. -A :class:`string ` is a special kind of sequence whose items are -*characters*. A character is not a separate data type but a -string of exactly one character. +Subscriptions may also be used as targets in :ref:`assignment ` or +:ref:`deletion ` statements. +In these cases, the interpreter will call the subscripted object's +:meth:`~object.__setitem__` or :meth:`~object.__delitem__` +:term:`special method`, respectively, instead of :meth:`~object.__getitem__`. +.. code-block:: -.. _slicings: + >>> colors = ['red', 'blue', 'green', 'black'] + >>> colors[3] = 'white' # Setting item at index + >>> colors + ['red', 'blue', 'green', 'white'] + >>> del colors[3] # Deleting item at index 3 + >>> colors + ['red', 'blue', 'green'] + +All advanced forms of *subscript* documented in the following sections +are also usable for assignment and deletion. -Slicings --------- .. index:: single: slicing @@ -1022,43 +1147,111 @@ Slicings pair: object; tuple pair: object; list -A slicing selects a range of items in a sequence object (e.g., a string, tuple -or list). Slicings may be used as expressions or as targets in assignment or -:keyword:`del` statements. The syntax for a slicing: +.. _slicings: -.. productionlist:: python-grammar - slicing: `primary` "[" `slice_list` "]" - slice_list: `slice_item` ("," `slice_item`)* [","] - slice_item: `expression` | `proper_slice` - proper_slice: [`lower_bound`] ":" [`upper_bound`] [ ":" [`stride`] ] - lower_bound: `expression` - upper_bound: `expression` - stride: `expression` +Slicings +^^^^^^^^ -There is ambiguity in the formal syntax here: anything that looks like an -expression list also looks like a slice list, so any subscription can be -interpreted as a slicing. Rather than further complicating the syntax, this is -disambiguated by defining that in this case the interpretation as a subscription -takes priority over the interpretation as a slicing (this is the case if the -slice list contains no proper slice). +A more advanced form of subscription, :dfn:`slicing`, is commonly used +to extract a portion of a :ref:`sequence `. +In this form, the subscript is a :term:`slice`: up to three +expressions separated by colons. +Any of the expressions may be omitted, but a slice must contain at least one +colon:: -.. index:: - single: start (slice object attribute) - single: stop (slice object attribute) - single: step (slice object attribute) + >>> number_names = ['zero', 'one', 'two', 'three', 'four', 'five'] + >>> number_names[1:3] + ['one', 'two'] + >>> number_names[1:] + ['one', 'two', 'three', 'four', 'five'] + >>> number_names[:3] + ['zero', 'one', 'two'] + >>> number_names[:] + ['zero', 'one', 'two', 'three', 'four', 'five'] + >>> number_names[::2] + ['zero', 'two', 'four'] + >>> number_names[:-3] + ['zero', 'one', 'two'] + >>> del number_names[4:] + >>> number_names + ['zero', 'one', 'two', 'three'] -The semantics for a slicing are as follows. The primary is indexed (using the -same :meth:`~object.__getitem__` method as -normal subscription) with a key that is constructed from the slice list, as -follows. If the slice list contains at least one comma, the key is a tuple -containing the conversion of the slice items; otherwise, the conversion of the -lone slice item is the key. The conversion of a slice item that is an -expression is that expression. The conversion of a proper slice is a slice -object (see section :ref:`types`) whose :attr:`~slice.start`, -:attr:`~slice.stop` and :attr:`~slice.step` attributes are the values of the -expressions given as lower bound, upper bound and stride, respectively, -substituting ``None`` for missing expressions. +When a slice is evaluated, the interpreter constructs a :class:`slice` object +whose :attr:`~slice.start`, :attr:`~slice.stop` and +:attr:`~slice.step` attributes, respectively, are the results of the +expressions between the colons. +Any missing expression evaluates to :const:`None`. +This :class:`!slice` object is then passed to the :meth:`~object.__getitem__` +or :meth:`~object.__class_getitem__` :term:`special method`, as above. :: + # continuing with the SubscriptionDemo instance defined above: + >>> demo[2:3] + subscripted with: slice(2, 3, None) + >>> demo[::'spam'] + subscripted with: slice(None, None, 'spam') + + +Comma-separated subscripts +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The subscript can also be given as two or more comma-separated expressions +or slices:: + + # continuing with the SubscriptionDemo instance defined above: + >>> demo[1, 2, 3] + subscripted with: (1, 2, 3) + >>> demo[1:2, 3] + subscripted with: (slice(1, 2, None), 3) + +This form is commonly used with numerical libraries for slicing +multi-dimensional data. +In this case, the interpreter constructs a :class:`tuple` of the results of the +expressions or slices, and passes this tuple to the :meth:`~object.__getitem__` +or :meth:`~object.__class_getitem__` :term:`special method`, as above. + +The subscript may also be given as a single expression or slice followed +by a comma, to specify a one-element tuple:: + + >>> demo['spam',] + subscripted with: ('spam',) + + +"Starred" subscriptions +^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 3.11 + Expressions in *tuple_slices* may be starred. See :pep:`646`. + +The subscript can also contain a starred expression. +In this case, the interpreter unpacks the result into a tuple, and passes +this tuple to :meth:`~object.__getitem__` or :meth:`~object.__class_getitem__`:: + + # continuing with the SubscriptionDemo instance defined above: + >>> demo[*range(10)] + subscripted with: (0, 1, 2, 3, 4, 5, 6, 7, 8, 9) + +Starred expressions may be combined with comma-separated expressions +and slices:: + + >>> demo['a', 'b', *range(3), 'c'] + subscripted with: ('a', 'b', 0, 1, 2, 'c') + + +Formal subscription grammar +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. grammar-snippet:: + :group: python-grammar + + subscription: `primary` '[' `subscript` ']' + subscript: `single_subscript` | `tuple_subscript` + single_subscript: `proper_slice` | `assignment_expression` + proper_slice: [`expression`] ":" [`expression`] [ ":" [`expression`] ] + tuple_subscript: ','.(`single_subscript` | `starred_expression`)+ [','] + +Recall that the ``|`` operator :ref:`denotes ordered choice `. +Specifically, in :token:`!subscript`, if both alternatives would match, the +first (:token:`!single_subscript`) has priority. .. index:: pair: object; callable @@ -1297,8 +1490,9 @@ for the operands): ``-1**2`` results in ``-1``. The power operator has the same semantics as the built-in :func:`pow` function, when called with two arguments: it yields its left argument raised to the power -of its right argument. The numeric arguments are first converted to a common -type, and the result is of that type. +of its right argument. +Numeric arguments are first :ref:`converted to a common type `, +and the result is of that type. For int operands, the result has the same type as the operands unless the second argument is negative; in that case, all arguments are converted to float and a @@ -1384,9 +1578,10 @@ operators and one for additive operators: The ``*`` (multiplication) operator yields the product of its arguments. The arguments must either both be numbers, or one argument must be an integer and -the other must be a sequence. In the former case, the numbers are converted to a -common real type and then multiplied together. In the latter case, sequence -repetition is performed; a negative repetition factor yields an empty sequence. +the other must be a sequence. In the former case, the numbers are +:ref:`converted to a common real type ` and then +multiplied together. In the latter case, sequence repetition is performed; +a negative repetition factor yields an empty sequence. This operation can be customized using the special :meth:`~object.__mul__` and :meth:`~object.__rmul__` methods. @@ -1414,7 +1609,8 @@ This operation can be customized using the special :meth:`~object.__matmul__` an pair: operator; // The ``/`` (division) and ``//`` (floor division) operators yield the quotient of -their arguments. The numeric arguments are first converted to a common type. +their arguments. The numeric arguments are first +:ref:`converted to a common type `. Division of integers yields a float, while floor division of integers results in an integer; the result is that of mathematical division with the 'floor' function applied to the result. Division by zero raises the :exc:`ZeroDivisionError` @@ -1430,8 +1626,9 @@ The floor division operation can be customized using the special pair: operator; % (percent) The ``%`` (modulo) operator yields the remainder from the division of the first -argument by the second. The numeric arguments are first converted to a common -type. A zero right argument raises the :exc:`ZeroDivisionError` exception. The +argument by the second. The numeric arguments are first +:ref:`converted to a common type `. +A zero right argument raises the :exc:`ZeroDivisionError` exception. The arguments may be floating-point numbers, e.g., ``3.14%0.7`` equals ``0.34`` (since ``3.14`` equals ``4*0.7 + 0.34``.) The modulo operator always yields a result with the same sign as its second operand (or zero); the absolute value of @@ -1462,7 +1659,9 @@ floating-point number using the :func:`abs` function if appropriate. The ``+`` (addition) operator yields the sum of its arguments. The arguments must either both be numbers or both be sequences of the same type. In the -former case, the numbers are converted to a common real type and then added together. +former case, the numbers are +:ref:`converted to a common real type ` and then +added together. In the latter case, the sequences are concatenated. This operation can be customized using the special :meth:`~object.__add__` and @@ -1477,8 +1676,9 @@ This operation can be customized using the special :meth:`~object.__add__` and single: operator; - (minus) single: - (minus); binary operator -The ``-`` (subtraction) operator yields the difference of its arguments. The -numeric arguments are first converted to a common real type. +The ``-`` (subtraction) operator yields the difference of its arguments. +The numeric arguments are first +:ref:`converted to a common real type `. This operation can be customized using the special :meth:`~object.__sub__` and :meth:`~object.__rsub__` methods. @@ -1938,8 +2138,9 @@ Conditional expressions conditional_expression: `or_test` ["if" `or_test` "else" `expression`] expression: `conditional_expression` | `lambda_expr` -Conditional expressions (sometimes called a "ternary operator") have the lowest -priority of all Python operations. +A conditional expression (sometimes called a "ternary operator") is an +alternative to the if-else statement. As it is an expression, it returns a value +and can appear as a sub-expression. The expression ``x if C else y`` first evaluates the condition, *C* rather than *x*. If *C* is true, *x* is evaluated and its value is returned; otherwise, *y* is @@ -2075,7 +2276,7 @@ precedence and have a left-to-right chaining feature as described in the | ``{key: value...}``, | dictionary display, | | ``{expressions...}`` | set display | +-----------------------------------------------+-------------------------------------+ -| ``x[index]``, ``x[index:index]``, | Subscription, slicing, | +| ``x[index]``, ``x[index:index]`` | Subscription (including slicing), | | ``x(arguments...)``, ``x.attribute`` | call, attribute reference | +-----------------------------------------------+-------------------------------------+ | :keyword:`await x ` | Await expression | diff --git a/Doc/reference/grammar.rst b/Doc/reference/grammar.rst index 1037feb691f..0ce8e42ddf3 100644 --- a/Doc/reference/grammar.rst +++ b/Doc/reference/grammar.rst @@ -12,8 +12,17 @@ The notation used here is the same as in the preceding docs, and is described in the :ref:`notation ` section, except for an extra complication: -* ``~`` ("cut"): commit to the current alternative and fail the rule - even if this fails to parse +* ``~`` ("cut"): commit to the current alternative; fail the rule + if the alternative fails to parse + + Python mainly uses cuts for optimizations or improved error + messages. They often appear to be useless in the listing below. + + .. see gh-143054, and CutValidator in the source, if you want to change this: + + Cuts currently don't appear inside parentheses, brackets, lookaheads + and similar. + Their behavior in these contexts is deliberately left unspecified. .. literalinclude:: ../../Grammar/python.gram :language: peg diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index d772d1f5345..83f0ee75e7a 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -359,21 +359,16 @@ of what happens during the loading portion of import:: if spec.loader is None: # unsupported raise ImportError - if spec.origin is None and spec.submodule_search_locations is not None: - # namespace package - sys.modules[spec.name] = module - elif not hasattr(spec.loader, 'exec_module'): - module = spec.loader.load_module(spec.name) - else: - sys.modules[spec.name] = module + + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + except BaseException: try: - spec.loader.exec_module(module) - except BaseException: - try: - del sys.modules[spec.name] - except KeyError: - pass - raise + del sys.modules[spec.name] + except KeyError: + pass + raise return sys.modules[spec.name] Note the following details: @@ -408,7 +403,10 @@ Note the following details: .. versionchanged:: 3.4 The import system has taken over the boilerplate responsibilities of loaders. These were previously performed by the - :meth:`importlib.abc.Loader.load_module` method. + ``importlib.abc.Loader.load_module`` method. + +.. versionchanged:: 3.15 + The ``load_module`` method is no longer used. Loaders ------- @@ -443,7 +441,7 @@ import machinery will create the new module itself. The :meth:`~importlib.abc.Loader.create_module` method of loaders. .. versionchanged:: 3.4 - The :meth:`~importlib.abc.Loader.load_module` method was replaced by + The ``importlib.abc.Loader.load_module`` method was replaced by :meth:`~importlib.abc.Loader.exec_module` and the import machinery assumed all the boilerplate responsibilities of loading. @@ -834,9 +832,7 @@ entirely with a custom meta path hook. If it is acceptable to only alter the behaviour of import statements without affecting other APIs that access the import system, then replacing -the builtin :func:`__import__` function may be sufficient. This technique -may also be employed at the module level to only alter the behaviour of -import statements within that module. +the builtin :func:`__import__` function may be sufficient. To selectively prevent the import of some modules from a hook early on the meta path (rather than disabling the standard import system entirely), diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index f93666dcdc8..f3ed1539493 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -10,12 +10,76 @@ Lexical analysis A Python program is read by a *parser*. Input to the parser is a stream of :term:`tokens `, generated by the *lexical analyzer* (also known as the *tokenizer*). -This chapter describes how the lexical analyzer breaks a file into tokens. +This chapter describes how the lexical analyzer produces these tokens. -Python reads program text as Unicode code points; the encoding of a source file -can be given by an encoding declaration and defaults to UTF-8, see :pep:`3120` -for details. If the source file cannot be decoded, a :exc:`SyntaxError` is -raised. +The lexical analyzer determines the program text's :ref:`encoding ` +(UTF-8 by default), and decodes the text into +:ref:`source characters `. +If the text cannot be decoded, a :exc:`SyntaxError` is raised. + +Next, the lexical analyzer uses the source characters to generate a stream of tokens. +The type of a generated token generally depends on the next source character to +be processed. Similarly, other special behavior of the analyzer depends on +the first source character that hasn't yet been processed. +The following table gives a quick summary of these source characters, +with links to sections that contain more information. + +.. list-table:: + :header-rows: 1 + + * - Character + - Next token (or other relevant documentation) + + * - * space + * tab + * formfeed + - * :ref:`Whitespace ` + + * - * CR, LF + - * :ref:`New line ` + * :ref:`Indentation ` + + * - * backslash (``\``) + - * :ref:`Explicit line joining ` + * (Also significant in :ref:`string escape sequences `) + + * - * hash (``#``) + - * :ref:`Comment ` + + * - * quote (``'``, ``"``) + - * :ref:`String literal ` + + * - * ASCII letter (``a``-``z``, ``A``-``Z``) + * non-ASCII character + - * :ref:`Name ` + * Prefixed :ref:`string or bytes literal ` + + * - * underscore (``_``) + - * :ref:`Name ` + * (Can also be part of :ref:`numeric literals `) + + * - * number (``0``-``9``) + - * :ref:`Numeric literal ` + + * - * dot (``.``) + - * :ref:`Numeric literal ` + * :ref:`Operator ` + + * - * question mark (``?``) + * dollar (``$``) + * + .. (the following uses zero-width space characters to render + .. a literal backquote) + + backquote (``​`​``) + * control character + - * Error (outside string literals and comments) + + * - * other printing character + - * :ref:`Operator or delimiter ` + + * - * end of file + - * :ref:`End marker ` .. _line-structure: @@ -120,6 +184,8 @@ If an encoding is declared, the encoding name must be recognized by Python encoding is used for all lexical analysis, including string literals, comments and identifiers. +.. _lexical-source-character: + All lexical analysis, including string literals, comments and identifiers, works on Unicode text decoded using the source encoding. Any Unicode code point, except the NUL control character, can appear in @@ -279,7 +345,15 @@ Whitespace between tokens Except at the beginning of a logical line or in string literals, the whitespace characters space, tab and formfeed can be used interchangeably to separate -tokens. Whitespace is needed between two tokens only if their concatenation +tokens: + +.. grammar-snippet:: + :group: python-grammar + + whitespace: ' ' | tab | formfeed + + +Whitespace is needed between two tokens only if their concatenation could otherwise be interpreted as a different token. For example, ``ab`` is one token, but ``a b`` is two tokens. However, ``+a`` and ``+ a`` both produce two tokens, ``+`` and ``a``, as ``+a`` is not a valid token. @@ -320,73 +394,29 @@ Names (identifiers and keywords) :data:`~token.NAME` tokens represent *identifiers*, *keywords*, and *soft keywords*. -Within the ASCII range (U+0001..U+007F), the valid characters for names -include the uppercase and lowercase letters (``A-Z`` and ``a-z``), -the underscore ``_`` and, except for the first character, the digits -``0`` through ``9``. +Names are composed of the following characters: + +* uppercase and lowercase letters (``A-Z`` and ``a-z``), +* the underscore (``_``), +* digits (``0`` through ``9``), which cannot appear as the first character, and +* non-ASCII characters. Valid names may only contain "letter-like" and + "digit-like" characters; see :ref:`lexical-names-nonascii` for details. Names must contain at least one character, but have no upper length limit. Case is significant. -Besides ``A-Z``, ``a-z``, ``_`` and ``0-9``, names can also use "letter-like" -and "number-like" characters from outside the ASCII range, as detailed below. - -All identifiers are converted into the `normalization form`_ NFKC while -parsing; comparison of identifiers is based on NFKC. - -Formally, the first character of a normalized identifier must belong to the -set ``id_start``, which is the union of: - -* Unicode category ```` - uppercase letters (includes ``A`` to ``Z``) -* Unicode category ```` - lowercase letters (includes ``a`` to ``z``) -* Unicode category ```` - titlecase letters -* Unicode category ```` - modifier letters -* Unicode category ```` - other letters -* Unicode category ```` - letter numbers -* {``"_"``} - the underscore -* ```` - an explicit set of characters in `PropList.txt`_ - to support backwards compatibility - -The remaining characters must belong to the set ``id_continue``, which is the -union of: - -* all characters in ``id_start`` -* Unicode category ```` - decimal numbers (includes ``0`` to ``9``) -* Unicode category ```` - connector punctuations -* Unicode category ```` - nonspacing marks -* Unicode category ```` - spacing combining marks -* ```` - another explicit set of characters in - `PropList.txt`_ to support backwards compatibility - -Unicode categories use the version of the Unicode Character Database as -included in the :mod:`unicodedata` module. - -These sets are based on the Unicode standard annex `UAX-31`_. -See also :pep:`3131` for further details. - -Even more formally, names are described by the following lexical definitions: +Formally, names are described by the following lexical definitions: .. grammar-snippet:: :group: python-grammar - NAME: `xid_start` `xid_continue`* - id_start: | | | | | | "_" | - id_continue: `id_start` | | | | | - xid_start: - xid_continue: - identifier: <`NAME`, except keywords> + NAME: `name_start` `name_continue`* + name_start: "a"..."z" | "A"..."Z" | "_" | + name_continue: name_start | "0"..."9" + identifier: <`NAME`, except keywords> -A non-normative listing of all valid identifier characters as defined by -Unicode is available in the `DerivedCoreProperties.txt`_ file in the Unicode -Character Database. - - -.. _UAX-31: https://www.unicode.org/reports/tr31/ -.. _PropList.txt: https://www.unicode.org/Public/16.0.0/ucd/PropList.txt -.. _DerivedCoreProperties.txt: https://www.unicode.org/Public/16.0.0/ucd/DerivedCoreProperties.txt -.. _normalization form: https://www.unicode.org/reports/tr15/#Norm_Forms +Note that not all names matched by this grammar are valid; see +:ref:`lexical-names-nonascii` for details. .. _keywords: @@ -427,6 +457,7 @@ Some names are only reserved under specific contexts. These are known as - ``match``, ``case``, and ``_``, when used in the :keyword:`match` statement. - ``type``, when used in the :keyword:`type` statement. +- ``lazy``, when used before an :keyword:`import` statement. These syntactically act as keywords in their specific contexts, but this distinction is done at the parser level, not when tokenizing. @@ -438,6 +469,9 @@ identifier names. .. versionchanged:: 3.12 ``type`` is now a soft keyword. +.. versionchanged:: 3.15 + ``lazy`` is now a soft keyword. + .. index:: single: _, identifiers single: __, identifiers @@ -489,6 +523,95 @@ characters: :ref:`atom-identifiers`. +.. _lexical-names-nonascii: + +Non-ASCII characters in names +----------------------------- + +Names that contain non-ASCII characters need additional normalization +and validation beyond the rules and grammar explained +:ref:`above `. +For example, ``ř_1``, ``蛇``, or ``साँप`` are valid names, but ``r〰2``, +``€``, or ``🐍`` are not. + +This section explains the exact rules. + +All names are converted into the `normalization form`_ NFKC while parsing. +This means that, for example, some typographic variants of characters are +converted to their "basic" form. For example, ``fiⁿₐˡᵢᶻₐᵗᵢᵒₙ`` normalizes to +``finalization``, so Python treats them as the same name:: + + >>> fiⁿₐˡᵢᶻₐᵗᵢᵒₙ = 3 + >>> finalization + 3 + +.. note:: + + Normalization is done at the lexical level only. + Run-time functions that take names as *strings* generally do not normalize + their arguments. + For example, the variable defined above is accessible at run time in the + :func:`globals` dictionary as ``globals()["finalization"]`` but not + ``globals()["fiⁿₐˡᵢᶻₐᵗᵢᵒₙ"]``. + +Similarly to how ASCII-only names must contain only letters, digits and +the underscore, and cannot start with a digit, a valid name must +start with a character in the "letter-like" set ``xid_start``, +and the remaining characters must be in the "letter- and digit-like" set +``xid_continue``. + +These sets are based on the *XID_Start* and *XID_Continue* sets as defined by the +Unicode standard annex `UAX-31`_. +Python's ``xid_start`` additionally includes the underscore (``_``). +Note that Python does not necessarily conform to `UAX-31`_. + +A non-normative listing of characters in the *XID_Start* and *XID_Continue* +sets as defined by Unicode is available in the `DerivedCoreProperties.txt`_ +file in the Unicode Character Database. +For reference, the construction rules for the ``xid_*`` sets are given below. + +The set ``id_start`` is defined as the union of: + +* Unicode category ```` - uppercase letters (includes ``A`` to ``Z``) +* Unicode category ```` - lowercase letters (includes ``a`` to ``z``) +* Unicode category ```` - titlecase letters +* Unicode category ```` - modifier letters +* Unicode category ```` - other letters +* Unicode category ```` - letter numbers +* {``"_"``} - the underscore +* ```` - an explicit set of characters in `PropList.txt`_ + to support backwards compatibility + +The set ``xid_start`` then closes this set under NFKC normalization, by +removing all characters whose normalization is not of the form +``id_start id_continue*``. + +The set ``id_continue`` is defined as the union of: + +* ``id_start`` (see above) +* Unicode category ```` - decimal numbers (includes ``0`` to ``9``) +* Unicode category ```` - connector punctuations +* Unicode category ```` - nonspacing marks +* Unicode category ```` - spacing combining marks +* ```` - another explicit set of characters in + `PropList.txt`_ to support backwards compatibility + +Again, ``xid_continue`` closes this set under NFKC normalization. + +Unicode categories use the version of the Unicode Character Database as +included in the :mod:`unicodedata` module. + +.. _UAX-31: https://www.unicode.org/reports/tr31/ +.. _PropList.txt: https://www.unicode.org/Public/17.0.0/ucd/PropList.txt +.. _DerivedCoreProperties.txt: https://www.unicode.org/Public/17.0.0/ucd/DerivedCoreProperties.txt +.. _normalization form: https://www.unicode.org/reports/tr15/#Norm_Forms + +.. seealso:: + + * :pep:`3131` -- Supporting Non-ASCII Identifiers + * :pep:`672` -- Unicode-related Security Considerations for Python + + .. _literals: Literals @@ -793,7 +916,7 @@ with the given *name*:: This sequence cannot appear in :ref:`bytes literals `. .. versionchanged:: 3.3 - Support for `name aliases `__ + Support for `name aliases `__ has been added. .. _string-escape-long-hex: @@ -921,124 +1044,59 @@ f-strings --------- .. versionadded:: 3.6 +.. versionchanged:: 3.7 + The :keyword:`await` and :keyword:`async for` can be used in expressions + within f-strings. +.. versionchanged:: 3.8 + Added the debug specifier (``=``) +.. versionchanged:: 3.12 + Many restrictions on expressions within f-strings have been removed. + Notably, nested strings, comments, and backslashes are now permitted. A :dfn:`formatted string literal` or :dfn:`f-string` is a string literal -that is prefixed with '``f``' or '``F``'. These strings may contain -replacement fields, which are expressions delimited by curly braces ``{}``. -While other string literals always have a constant value, formatted strings -are really expressions evaluated at run time. +that is prefixed with '``f``' or '``F``'. +Unlike other string literals, f-strings do not have a constant value. +They may contain *replacement fields* delimited by curly braces ``{}``. +Replacement fields contain expressions which are evaluated at run time. +For example:: -Escape sequences are decoded like in ordinary string literals (except when -a literal is also marked as a raw string). After decoding, the grammar -for the contents of the string is: + >>> who = 'nobody' + >>> nationality = 'Spanish' + >>> f'{who.title()} expects the {nationality} Inquisition!' + 'Nobody expects the Spanish Inquisition!' -.. productionlist:: python-grammar - f_string: (`literal_char` | "{{" | "}}" | `replacement_field`)* - replacement_field: "{" `f_expression` ["="] ["!" `conversion`] [":" `format_spec`] "}" - f_expression: (`conditional_expression` | "*" `or_expr`) - : ("," `conditional_expression` | "," "*" `or_expr`)* [","] - : | `yield_expression` - conversion: "s" | "r" | "a" - format_spec: (`literal_char` | `replacement_field`)* - literal_char: +Any doubled curly braces (``{{`` or ``}}``) outside replacement fields +are replaced with the corresponding single curly brace:: -The parts of the string outside curly braces are treated literally, -except that any doubled curly braces ``'{{'`` or ``'}}'`` are replaced -with the corresponding single curly brace. A single opening curly -bracket ``'{'`` marks a replacement field, which starts with a -Python expression. To display both the expression text and its value after -evaluation, (useful in debugging), an equal sign ``'='`` may be added after the -expression. A conversion field, introduced by an exclamation point ``'!'`` may -follow. A format specifier may also be appended, introduced by a colon ``':'``. -A replacement field ends with a closing curly bracket ``'}'``. + >>> print(f'{{...}}') + {...} + +Other characters outside replacement fields are treated like in ordinary +string literals. +This means that escape sequences are decoded (except when a literal is +also marked as a raw string), and newlines are possible in triple-quoted +f-strings:: + + >>> name = 'Galahad' + >>> favorite_color = 'blue' + >>> print(f'{name}:\t{favorite_color}') + Galahad: blue + >>> print(rf"C:\Users\{name}") + C:\Users\Galahad + >>> print(f'''Three shall be the number of the counting + ... and the number of the counting shall be three.''') + Three shall be the number of the counting + and the number of the counting shall be three. Expressions in formatted string literals are treated like regular -Python expressions surrounded by parentheses, with a few exceptions. -An empty expression is not allowed, and both :keyword:`lambda` and -assignment expressions ``:=`` must be surrounded by explicit parentheses. +Python expressions. Each expression is evaluated in the context where the formatted string literal -appears, in order from left to right. Replacement expressions can contain -newlines in both single-quoted and triple-quoted f-strings and they can contain -comments. Everything that comes after a ``#`` inside a replacement field -is a comment (even closing braces and quotes). In that case, replacement fields -must be closed in a different line. - -.. code-block:: text - - >>> f"abc{a # This is a comment }" - ... + 3}" - 'abc5' - -.. versionchanged:: 3.7 - Prior to Python 3.7, an :keyword:`await` expression and comprehensions - containing an :keyword:`async for` clause were illegal in the expressions - in formatted string literals due to a problem with the implementation. - -.. versionchanged:: 3.12 - Prior to Python 3.12, comments were not allowed inside f-string replacement - fields. - -When the equal sign ``'='`` is provided, the output will have the expression -text, the ``'='`` and the evaluated value. Spaces after the opening brace -``'{'``, within the expression and after the ``'='`` are all retained in the -output. By default, the ``'='`` causes the :func:`repr` of the expression to be -provided, unless there is a format specified. When a format is specified it -defaults to the :func:`str` of the expression unless a conversion ``'!r'`` is -declared. - -.. versionadded:: 3.8 - The equal sign ``'='``. - -If a conversion is specified, the result of evaluating the expression -is converted before formatting. Conversion ``'!s'`` calls :func:`str` on -the result, ``'!r'`` calls :func:`repr`, and ``'!a'`` calls :func:`ascii`. - -The result is then formatted using the :func:`format` protocol. The -format specifier is passed to the :meth:`~object.__format__` method of the -expression or conversion result. An empty string is passed when the -format specifier is omitted. The formatted result is then included in -the final value of the whole string. - -Top-level format specifiers may include nested replacement fields. These nested -fields may include their own conversion fields and :ref:`format specifiers -`, but may not include more deeply nested replacement fields. The -:ref:`format specifier mini-language ` is the same as that used by -the :meth:`str.format` method. - -Formatted string literals may be concatenated, but replacement fields -cannot be split across literals. - -Some examples of formatted string literals:: - - >>> name = "Fred" - >>> f"He said his name is {name!r}." - "He said his name is 'Fred'." - >>> f"He said his name is {repr(name)}." # repr() is equivalent to !r - "He said his name is 'Fred'." - >>> width = 10 - >>> precision = 4 - >>> value = decimal.Decimal("12.34567") - >>> f"result: {value:{width}.{precision}}" # nested fields - 'result: 12.35' - >>> today = datetime(year=2017, month=1, day=27) - >>> f"{today:%B %d, %Y}" # using date format specifier - 'January 27, 2017' - >>> f"{today=:%B %d, %Y}" # using date format specifier and debugging - 'today=January 27, 2017' - >>> number = 1024 - >>> f"{number:#0x}" # using integer format specifier - '0x400' - >>> foo = "bar" - >>> f"{ foo = }" # preserves whitespace - " foo = 'bar'" - >>> line = "The mill's closed" - >>> f"{line = }" - 'line = "The mill\'s closed"' - >>> f"{line = :20}" - "line = The mill's closed " - >>> f"{line = !r:20}" - 'line = "The mill\'s closed" ' +appears, in order from left to right. +An empty expression is not allowed, and both :keyword:`lambda` and +assignment expressions ``:=`` must be surrounded by explicit parentheses:: + >>> f'{(half := 1/2)}, {half * 42}' + '0.5, 21.0' Reusing the outer f-string quoting type inside a replacement field is permitted:: @@ -1047,10 +1105,6 @@ permitted:: >>> f"abc {a["x"]} def" 'abc 2 def' -.. versionchanged:: 3.12 - Prior to Python 3.12, reuse of the same quoting type of the outer f-string - inside a replacement field was not possible. - Backslashes are also allowed in replacement fields and are evaluated the same way as in any other context:: @@ -1061,23 +1115,84 @@ way as in any other context:: b c -.. versionchanged:: 3.12 - Prior to Python 3.12, backslashes were not permitted inside an f-string - replacement field. +It is possible to nest f-strings:: -Formatted string literals cannot be used as docstrings, even if they do not -include expressions. + >>> name = 'world' + >>> f'Repeated:{f' hello {name}' * 3}' + 'Repeated: hello world hello world hello world' -:: +Portable Python programs should not use more than 5 levels of nesting. + +.. impl-detail:: + + CPython does not limit nesting of f-strings. + +Replacement expressions can contain newlines in both single-quoted and +triple-quoted f-strings and they can contain comments. +Everything that comes after a ``#`` inside a replacement field +is a comment (even closing braces and quotes). +This means that replacement fields with comments must be closed in a +different line: + +.. code-block:: text + + >>> a = 2 + >>> f"abc{a # This comment }" continues until the end of the line + ... + 3}" + 'abc5' + +After the expression, replacement fields may optionally contain: + +* a *debug specifier* -- an equal sign (``=``), optionally surrounded by + whitespace on one or both sides; +* a *conversion specifier* -- ``!s``, ``!r`` or ``!a``; and/or +* a *format specifier* prefixed with a colon (``:``). + +See the :ref:`Standard Library section on f-strings ` +for details on how these fields are evaluated. + +As that section explains, *format specifiers* are passed as the second argument +to the :func:`format` function to format a replacement field value. +For example, they can be used to specify a field width and padding characters +using the :ref:`Format Specification Mini-Language `:: + + >>> number = 14.3 + >>> f'{number:20.7f}' + ' 14.3000000' + +Top-level format specifiers may include nested replacement fields:: + + >>> field_size = 20 + >>> precision = 7 + >>> f'{number:{field_size}.{precision}f}' + ' 14.3000000' + +These nested fields may include their own conversion fields and +:ref:`format specifiers `:: + + >>> number = 3 + >>> f'{number:{field_size}}' + ' 3' + >>> f'{number:{field_size:05}}' + '00000000000000000003' + +However, these nested fields may not include more deeply nested replacement +fields. + +Formatted string literals cannot be used as :term:`docstrings `, +even if they do not include expressions:: >>> def foo(): ... f"Not a docstring" ... - >>> foo.__doc__ is None - True + >>> print(foo.__doc__) + None -See also :pep:`498` for the proposal that added formatted string literals, -and :meth:`str.format`, which uses a related format string mechanism. +.. seealso:: + + * :pep:`498` -- Literal String Interpolation + * :pep:`701` -- Syntactic formalization of f-strings + * :meth:`str.format`, which uses a related format string mechanism. .. _t-strings: @@ -1090,36 +1205,99 @@ t-strings A :dfn:`template string literal` or :dfn:`t-string` is a string literal that is prefixed with '``t``' or '``T``'. -These strings follow the same syntax and evaluation rules as -:ref:`formatted string literals `, with the following differences: +These strings follow the same syntax rules as +:ref:`formatted string literals `. +For differences in evaluation rules, see the +:ref:`Standard Library section on t-strings ` -* Rather than evaluating to a ``str`` object, template string literals evaluate - to a :class:`string.templatelib.Template` object. -* The :func:`format` protocol is not used. - Instead, the format specifier and conversions (if any) are passed to - a new :class:`~string.templatelib.Interpolation` object that is created - for each evaluated expression. - It is up to code that processes the resulting :class:`~string.templatelib.Template` - object to decide how to handle format specifiers and conversions. +Formal grammar for f-strings +---------------------------- -* Format specifiers containing nested replacement fields are evaluated eagerly, - prior to being passed to the :class:`~string.templatelib.Interpolation` object. - For instance, an interpolation of the form ``{amount:.{precision}f}`` will - evaluate the inner expression ``{precision}`` to determine the value of the - ``format_spec`` attribute. - If ``precision`` were to be ``2``, the resulting format specifier - would be ``'.2f'``. +F-strings are handled partly by the :term:`lexical analyzer`, which produces the +tokens :py:data:`~token.FSTRING_START`, :py:data:`~token.FSTRING_MIDDLE` +and :py:data:`~token.FSTRING_END`, and partly by the parser, which handles +expressions in the replacement field. +The exact way the work is split is a CPython implementation detail. -* When the equals sign ``'='`` is provided in an interpolation expression, - the text of the expression is appended to the literal string that precedes - the relevant interpolation. - This includes the equals sign and any surrounding whitespace. - The :class:`!Interpolation` instance for the expression will be created as - normal, except that :attr:`~string.templatelib.Interpolation.conversion` will - be set to '``r``' (:func:`repr`) by default. - If an explicit conversion or format specifier are provided, - this will override the default behaviour. +Correspondingly, the f-string grammar is a mix of +:ref:`lexical and syntactic definitions `. + +Whitespace is significant in these situations: + +* There may be no whitespace in :py:data:`~token.FSTRING_START` (between + the prefix and quote). +* Whitespace in :py:data:`~token.FSTRING_MIDDLE` is part of the literal + string contents. +* In ``fstring_replacement_field``, if ``f_debug_specifier`` is present, + all whitespace after the opening brace until the ``f_debug_specifier``, + as well as whitespace immediately following ``f_debug_specifier``, + is retained as part of the expression. + + .. impl-detail:: + + The expression is not handled in the tokenization phase; it is + retrieved from the source code using locations of the ``{`` token + and the token after ``=``. + + +The ``FSTRING_MIDDLE`` definition uses +:ref:`negative lookaheads ` (``!``) +to indicate special characters (backslash, newline, ``{``, ``}``) and +sequences (``f_quote``). + +.. grammar-snippet:: + :group: python-grammar + + fstring: `FSTRING_START` `fstring_middle`* `FSTRING_END` + + FSTRING_START: `fstringprefix` ("'" | '"' | "'''" | '"""') + FSTRING_END: `f_quote` + fstringprefix: <("f" | "fr" | "rf"), case-insensitive> + f_debug_specifier: '=' + f_quote: + + fstring_middle: + | `fstring_replacement_field` + | `FSTRING_MIDDLE` + FSTRING_MIDDLE: + | (!"\" !`newline` !'{' !'}' !`f_quote`) `source_character` + | `stringescapeseq` + | "{{" + | "}}" + | + fstring_replacement_field: + | '{' `f_expression` [`f_debug_specifier`] [`fstring_conversion`] + [`fstring_full_format_spec`] '}' + fstring_conversion: + | "!" ("s" | "r" | "a") + fstring_full_format_spec: + | ':' `fstring_format_spec`* + fstring_format_spec: + | `FSTRING_MIDDLE` + | `fstring_replacement_field` + f_expression: + | ','.(`conditional_expression` | "*" `or_expr`)+ [","] + | `yield_expression` + +.. note:: + + In the above grammar snippet, the ``f_quote`` and ``FSTRING_MIDDLE`` rules + are context-sensitive -- they depend on the contents of ``FSTRING_START`` + of the nearest enclosing ``fstring``. + + Constructing a more traditional formal grammar from this template is left + as an exercise for the reader. + +The grammar for t-strings is identical to the one for f-strings, with *t* +instead of *f* at the beginning of rule and token names and in the prefix. + +.. grammar-snippet:: + :group: python-grammar + + tstring: TSTRING_START tstring_middle* TSTRING_END + + .. _numbers: @@ -1242,8 +1420,8 @@ The parts are separated by a decimal point, ``.``:: 2.71828 4.0 -Unlike in integer literals, leading zeros are allowed in the numeric parts. -For example, ``077.010`` is legal, and denotes the same number as ``77.10``. +Unlike in integer literals, leading zeros are allowed. +For example, ``077.010`` is legal, and denotes the same number as ``77.01``. As in integer literals, single underscores may occur between digits to help readability:: diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 5831bd27cba..648e3a9bf54 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -91,11 +91,10 @@ attributes or items of mutable objects: : | "[" [`target_list`] "]" : | `attributeref` : | `subscription` - : | `slicing` : | "*" `target` -(See section :ref:`primaries` for the syntax definitions for *attributeref*, -*subscription*, and *slicing*.) +(See section :ref:`primaries` for the syntax definitions for *attributeref* +and *subscription*.) An assignment statement evaluates the expression list (remember that this can be a single expression or a comma-separated list, the latter yielding a tuple) and @@ -107,8 +106,8 @@ right. pair: target; list Assignment is defined recursively depending on the form of the target (list). -When a target is part of a mutable object (an attribute reference, subscription -or slicing), the mutable object must ultimately perform the assignment and +When a target is part of a mutable object (an attribute reference or +subscription), the mutable object must ultimately perform the assignment and decide about its validity, and may raise an exception if the assignment is unacceptable. The rules observed by various types and the exceptions raised are given with the definition of the object types (see section :ref:`types`). @@ -189,9 +188,14 @@ Assignment of an object to a single target is recursively defined as follows. pair: object; mutable * If the target is a subscription: The primary expression in the reference is - evaluated. It should yield either a mutable sequence object (such as a list) - or a mapping object (such as a dictionary). Next, the subscript expression is evaluated. + Next, the subscript expression is evaluated. + Then, the primary's :meth:`~object.__setitem__` method is called with + two arguments: the subscript and the assigned object. + + Typically, :meth:`~object.__setitem__` is defined on mutable sequence objects + (such as lists) and mapping objects (such as dictionaries), and behaves as + follows. .. index:: pair: object; sequence @@ -214,16 +218,13 @@ Assignment of an object to a single target is recursively defined as follows. object. This can either replace an existing key/value pair with the same key value, or insert a new key/value pair (if no key with the same value existed). - For user-defined objects, the :meth:`~object.__setitem__` method is called with - appropriate arguments. - .. index:: pair: slicing; assignment -* If the target is a slicing: The primary expression in the reference is - evaluated. It should yield a mutable sequence object (such as a list). The - assigned object should be a sequence object of the same type. Next, the lower - and upper bound expressions are evaluated, insofar they are present; defaults - are zero and the sequence's length. The bounds should evaluate to integers. + If the target is a slicing: The primary expression should evaluate to + a mutable sequence object (such as a list). + The assigned object should be :term:`iterable`. + The slicing's lower and upper bounds should be integers; if they are ``None`` + (or not present), the defaults are zero and the sequence's length. If either bound is negative, the sequence's length is added to it. The resulting bounds are clipped to lie between zero and the sequence's length, inclusive. Finally, the sequence object is asked to replace the slice with @@ -231,12 +232,6 @@ Assignment of an object to a single target is recursively defined as follows. from the length of the assigned sequence, thus changing the length of the target sequence, if the target sequence allows it. -.. impl-detail:: - - In the current implementation, the syntax for targets is taken to be the same - as for expressions, and invalid syntax is rejected during the code generation - phase, causing less detailed error messages. - Although the definition of assignment implies that overlaps between the left-hand side and the right-hand side are 'simultaneous' (for example ``a, b = b, a`` swaps two variables), overlaps *within* the collection of assigned-to @@ -281,7 +276,7 @@ operation and an assignment statement: .. productionlist:: python-grammar augmented_assignment_stmt: `augtarget` `augop` (`expression_list` | `yield_expression`) - augtarget: `identifier` | `attributeref` | `subscription` | `slicing` + augtarget: `identifier` | `attributeref` | `subscription` augop: "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**=" : | ">>=" | "<<=" | "&=" | "^=" | "|=" @@ -465,12 +460,12 @@ Deletion of a target list recursively deletes each target, from left to right. Deletion of a name removes the binding of that name from the local or global namespace, depending on whether the name occurs in a :keyword:`global` statement -in the same code block. If the name is unbound, a :exc:`NameError` exception -will be raised. +in the same code block. Trying to delete an unbound name raises a +:exc:`NameError` exception. .. index:: pair: attribute; deletion -Deletion of attribute references, subscriptions and slicings is passed to the +Deletion of attribute references and subscriptions is passed to the primary object involved; deletion of a slicing is in general equivalent to assignment of an empty slice of the right type (but even this is determined by the sliced object). @@ -748,14 +743,15 @@ The :keyword:`!import` statement pair: name; binding pair: keyword; from pair: keyword; as + pair: keyword; lazy pair: exception; ImportError single: , (comma); import statement .. productionlist:: python-grammar - import_stmt: "import" `module` ["as" `identifier`] ("," `module` ["as" `identifier`])* - : | "from" `relative_module` "import" `identifier` ["as" `identifier`] + import_stmt: ["lazy"] "import" `module` ["as" `identifier`] ("," `module` ["as" `identifier`])* + : | ["lazy"] "from" `relative_module` "import" `identifier` ["as" `identifier`] : ("," `identifier` ["as" `identifier`])* - : | "from" `relative_module` "import" "(" `identifier` ["as" `identifier`] + : | ["lazy"] "from" `relative_module` "import" "(" `identifier` ["as" `identifier`] : ("," `identifier` ["as" `identifier`])* [","] ")" : | "from" `relative_module` "import" "*" module: (`identifier` ".")* `identifier` @@ -836,7 +832,9 @@ where the :keyword:`import` statement occurs. The *public names* defined by a module are determined by checking the module's namespace for a variable named ``__all__``; if defined, it must be a sequence -of strings which are names defined or imported by that module. The names +of strings which are names defined or imported by that module. +Names containing non-ASCII characters must be in the `normalization form`_ +NFKC; see :ref:`lexical-names-nonascii` for details. The names given in ``__all__`` are all considered public and are required to exist. If ``__all__`` is not defined, the set of public names includes all names found in the module's namespace which do not begin with an underscore character @@ -870,6 +868,108 @@ determine dynamically the modules to be loaded. .. audit-event:: import module,filename,sys.path,sys.meta_path,sys.path_hooks import +.. _normalization form: https://www.unicode.org/reports/tr15/#Norm_Forms + +.. _lazy-imports: +.. _lazy: + +Lazy imports +------------ + +.. index:: + pair: lazy; import + single: lazy import + +The :keyword:`lazy` keyword is a :ref:`soft keyword ` that +only has special meaning when it appears immediately before an +:keyword:`import` or :keyword:`from` statement. When an import statement is +preceded by the :keyword:`lazy` keyword, the import becomes *lazy*: the +module is not loaded immediately at the import statement. Instead, a lazy +proxy object is created and bound to the name. The actual module is loaded +on first use of that name. + +Lazy imports are only permitted at module scope. Using :keyword:`lazy` +inside a function, class body, or +:keyword:`try`/:keyword:`except`/:keyword:`finally` block raises a +:exc:`SyntaxError`. Star imports cannot be lazy (``lazy from module import +*`` is a syntax error), and :ref:`future statements ` cannot be +lazy. + +When using ``lazy from ... import``, each imported name is bound to a lazy +proxy object. The first access to any of these names triggers loading of the +entire module and resolves only that specific name to its actual value. +Other names remain as lazy proxies until they are accessed. + +Example:: + + lazy import json + import sys + + print('json' in sys.modules) # False - json module not yet loaded + + # First use triggers loading + result = json.dumps({"hello": "world"}) + + print('json' in sys.modules) # True - now loaded + +If an error occurs during module loading (such as :exc:`ImportError` or +:exc:`SyntaxError`), it is raised at the point where the lazy import is first +used, not at the import statement itself. + +See :pep:`810` for the full specification of lazy imports. + +.. versionadded:: 3.15 + +.. _lazy-modules-compat: + +Compatibility via ``__lazy_modules__`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. index:: + single: __lazy_modules__ + +As an alternative to using the :keyword:`lazy` keyword, a module can opt +into lazy loading for specific imports by defining a module-level +:attr:`~module.__lazy_modules__` variable. When present, it must be a +container of fully qualified module name strings. Any regular (non-``lazy``) +:keyword:`import` statement at module scope whose target appears in +:attr:`!__lazy_modules__` is treated as a lazy import, exactly as if the +:keyword:`lazy` keyword had been used. + +This provides a way to enable lazy loading for specific dependencies without +changing individual ``import`` statements. This is useful when supporting +Python versions older than 3.15 while using lazy imports in 3.15+:: + + __lazy_modules__ = ["json", "pathlib"] + + import json # loaded lazily (name is in __lazy_modules__) + import os # loaded eagerly (name not in __lazy_modules__) + + import pathlib # loaded lazily + +Relative imports are resolved to their absolute name before the lookup, so +:attr:`!__lazy_modules__` must always contain fully qualified module names. + +For ``from``-style imports, the relevant name is the module following +``from``, not the names of its members:: + + # In mypackage/mymodule.py + __lazy_modules__ = ["mypackage", "mypackage.sub.utils"] + + from . import helper # loaded lazily: . resolves to mypackage + from .sub.utils import func # loaded lazily: .sub.utils resolves to mypackage.sub.utils + import json # loaded eagerly (not in __lazy_modules__) + +Imports inside functions, class bodies, or +:keyword:`try`/:keyword:`except`/:keyword:`finally` blocks are always eager, +regardless of :attr:`!__lazy_modules__`. + +Setting ``-X lazy_imports=none`` (or the :envvar:`PYTHON_LAZY_IMPORTS` +environment variable to ``none``) overrides :attr:`!__lazy_modules__` and +forces all imports to be eager. + +.. versionadded:: 3.15 + .. _future: Future statements diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 7b7286429a1..536ae57e4ef 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -7,15 +7,17 @@ # won't suddenly cause build failures. Updating the version is fine as long # as no warnings are raised by doing so. # Keep this version in sync with ``Doc/conf.py``. -sphinx~=8.2.0 +sphinx<9.0.0 blurb -sphinxext-opengraph~=0.12.0 +sphinxext-opengraph~=0.13.0 sphinx-notfound-page~=1.0.0 # The theme used by the documentation is stored separately, so we need # to install that as well. python-docs-theme>=2023.3.1,!=2023.7 +linklint + -c constraints.txt diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 0ee92dce437..189173a5f8a 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -2,14 +2,9 @@ # as tested on the CI via check-warnings.py in reusable-docs.yml. # Keep lines sorted lexicographically to help avoid merge conflicts. -Doc/c-api/descriptor.rst -Doc/c-api/float.rst -Doc/c-api/init.rst Doc/c-api/init_config.rst Doc/c-api/intro.rst -Doc/c-api/module.rst Doc/c-api/stable.rst -Doc/c-api/type.rst Doc/c-api/typeobj.rst Doc/library/ast.rst Doc/library/asyncio-extending.rst @@ -26,23 +21,16 @@ Doc/library/multiprocessing.rst Doc/library/optparse.rst Doc/library/os.rst Doc/library/pickletools.rst -Doc/library/profile.rst Doc/library/pyexpat.rst -Doc/library/resource.rst Doc/library/select.rst -Doc/library/signal.rst -Doc/library/smtplib.rst Doc/library/socket.rst Doc/library/ssl.rst -Doc/library/stdtypes.rst -Doc/library/subprocess.rst Doc/library/termios.rst Doc/library/test.rst Doc/library/tkinter.rst Doc/library/tkinter.scrolledtext.rst Doc/library/tkinter.ttk.rst Doc/library/unittest.mock.rst -Doc/library/unittest.rst Doc/library/urllib.parse.rst Doc/library/urllib.request.rst Doc/library/wsgiref.rst @@ -53,7 +41,6 @@ Doc/library/xml.sax.reader.rst Doc/library/xml.sax.rst Doc/library/xmlrpc.client.rst Doc/library/xmlrpc.server.rst -Doc/library/zlib.rst Doc/whatsnew/2.4.rst Doc/whatsnew/2.5.rst Doc/whatsnew/2.6.rst diff --git a/Doc/tools/check-html-ids.py b/Doc/tools/check-html-ids.py new file mode 100644 index 00000000000..7d86c6cc326 --- /dev/null +++ b/Doc/tools/check-html-ids.py @@ -0,0 +1,182 @@ +from compression import gzip +import concurrent.futures +from pathlib import Path +import html.parser +import functools +import argparse +import json +import sys +import re + + +IGNORED_ID_RE = re.compile( + r""" + index-\d+ + | id\d+ + | [_a-z]+_\d+ +""", + re.VERBOSE, +) + + +class IDGatherer(html.parser.HTMLParser): + def __init__(self, ids): + super().__init__() + self.__ids = ids + + def handle_starttag(self, tag, attrs): + for name, value in attrs: + if name == 'id': + if not IGNORED_ID_RE.fullmatch(value): + self.__ids.add(value) + + +def get_ids_from_file(path): + ids = set() + gatherer = IDGatherer(ids) + with path.open(encoding='utf-8') as file: + while chunk := file.read(4096): + gatherer.feed(chunk) + return ids + + +def gather_ids(htmldir, *, verbose_print): + if not htmldir.joinpath('objects.inv').exists(): + raise ValueError(f'{htmldir!r} is not a Sphinx HTML output directory') + + if sys._is_gil_enabled: + pool = concurrent.futures.ProcessPoolExecutor() + else: + pool = concurrent.futures.ThreadPoolExecutor() + tasks = {} + for path in htmldir.glob('**/*.html'): + relative_path = path.relative_to(htmldir) + if '_static' in relative_path.parts: + continue + if 'whatsnew' in relative_path.parts: + continue + tasks[relative_path] = pool.submit(get_ids_from_file, path=path) + + ids_by_page = {} + for relative_path, future in tasks.items(): + verbose_print(relative_path) + ids = future.result() + ids_by_page[str(relative_path)] = ids + verbose_print(f' - {len(ids)} ids found') + + common = set.intersection(*ids_by_page.values()) + verbose_print(f'Filtering out {len(common)} common ids') + for key, page_ids in ids_by_page.items(): + ids_by_page[key] = sorted(page_ids - common) + + return ids_by_page + + +def do_check(baseline, checked, excluded, *, verbose_print): + successful = True + for name, baseline_ids in sorted(baseline.items()): + try: + checked_ids = checked[name] + except KeyError: + successful = False + print(f'{name}: (page missing)') + print() + else: + missing_ids = set(baseline_ids) - set(checked_ids) + if missing_ids: + missing_ids = { + a + for a in missing_ids + if not IGNORED_ID_RE.fullmatch(a) + and (name, a) not in excluded + } + if missing_ids: + successful = False + for missing_id in sorted(missing_ids): + print(f'{name}: {missing_id}') + print() + return successful + + +def main(argv): + parser = argparse.ArgumentParser() + parser.add_argument( + '-v', + '--verbose', + action='store_true', + help='print out more information', + ) + subparsers = parser.add_subparsers(dest='command', required=True) + + collect = subparsers.add_parser( + 'collect', help='collect IDs from a set of HTML files' + ) + collect.add_argument( + 'htmldir', type=Path, help='directory with HTML documentation' + ) + collect.add_argument( + '-o', + '--outfile', + help='File to save the result in; default /html-ids.json.gz', + ) + + check = subparsers.add_parser('check', help='check two archives of IDs') + check.add_argument( + 'baseline_file', type=Path, help='file with baseline IDs' + ) + check.add_argument('checked_file', type=Path, help='file with checked IDs') + check.add_argument( + '-x', + '--exclude-file', + type=Path, + help='file with IDs to exclude from the check', + ) + + args = parser.parse_args(argv[1:]) + + if args.verbose: + verbose_print = functools.partial(print, file=sys.stderr) + else: + + def verbose_print(*args, **kwargs): + """do nothing""" + + if args.command == 'collect': + ids = gather_ids(args.htmldir, verbose_print=verbose_print) + if args.outfile is None: + args.outfile = args.htmldir / 'html-ids.json.gz' + with gzip.open(args.outfile, 'wt', encoding='utf-8') as zfile: + json.dump({'ids_by_page': ids}, zfile) + + if args.command == 'check': + with gzip.open(args.baseline_file) as zfile: + baseline = json.load(zfile)['ids_by_page'] + with gzip.open(args.checked_file) as zfile: + checked = json.load(zfile)['ids_by_page'] + excluded = set() + if args.exclude_file: + with open(args.exclude_file, encoding='utf-8') as file: + for line in file: + line = line.strip() + if line and not line.startswith('#'): + name, sep, excluded_id = line.partition(':') + if sep: + excluded.add((name.strip(), excluded_id.strip())) + if do_check(baseline, checked, excluded, verbose_print=verbose_print): + verbose_print('All OK') + else: + sys.stdout.flush() + print( + 'ERROR: Removed IDs found', + 'The above HTML IDs were removed from the documentation, ' + + 'resulting in broken links. Please add them back.', + sep='\n', + file=sys.stderr, + ) + if args.exclude_file: + print(f'Alternatively, add them to {args.exclude_file}.') + sys.exit(1) + + +if __name__ == '__main__': + main(sys.argv) diff --git a/Doc/tools/check-warnings.py b/Doc/tools/check-warnings.py index 2f2bb9e2dcb..859bc1e2d5f 100644 --- a/Doc/tools/check-warnings.py +++ b/Doc/tools/check-warnings.py @@ -311,8 +311,11 @@ def main(argv: list[str] | None = None) -> int: if not Path("Doc").exists() or not Path("Doc").is_dir(): raise RuntimeError(wrong_directory_msg) - with Path("Doc/sphinx-warnings.txt").open(encoding="UTF-8") as f: - warnings = f.read().splitlines() + warnings = ( + Path("Doc/sphinx-warnings.txt") + .read_text(encoding="UTF-8") + .splitlines() + ) cwd = str(Path.cwd()) + os.path.sep files_with_nits = { diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index 089614a1f6c..1409c77aed9 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -3,10 +3,12 @@ * Reference count annotations for C API functions. * Stable ABI annotations * Limited API annotations +* Thread safety annotations for C API functions. Configuration: * Set ``refcount_file`` to the path to the reference count data file. * Set ``stable_abi_file`` to the path to stable ABI list. +* Set ``threadsafety_file`` to the path to the thread safety data file. """ from __future__ import annotations @@ -48,6 +50,15 @@ class RefCountEntry: result_refs: int | None = None +@dataclasses.dataclass(frozen=True, slots=True) +class ThreadSafetyEntry: + # Name of the function. + name: str + # Thread safety level. + # One of: 'incompatible', 'compatible', 'safe'. + level: str + + @dataclasses.dataclass(frozen=True, slots=True) class StableABIEntry: # Role of the object. @@ -113,10 +124,42 @@ def read_stable_abi_data(stable_abi_file: Path) -> dict[str, StableABIEntry]: return stable_abi_data +_VALID_THREADSAFETY_LEVELS = frozenset({ + "incompatible", + "compatible", + "distinct", + "shared", + "atomic", +}) + + +def read_threadsafety_data( + threadsafety_filename: Path, +) -> dict[str, ThreadSafetyEntry]: + threadsafety_data = {} + for line in threadsafety_filename.read_text(encoding="utf8").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + # Each line is of the form: function_name : level : [comment] + parts = line.split(":", 2) + if len(parts) < 2: + raise ValueError(f"Wrong field count in {line!r}") + name, level = parts[0].strip(), parts[1].strip() + if level not in _VALID_THREADSAFETY_LEVELS: + raise ValueError( + f"Unknown thread safety level {level!r} for {name!r}. " + f"Valid levels: {sorted(_VALID_THREADSAFETY_LEVELS)}" + ) + threadsafety_data[name] = ThreadSafetyEntry(name=name, level=level) + return threadsafety_data + + def add_annotations(app: Sphinx, doctree: nodes.document) -> None: state = app.env.domaindata["c_annotations"] refcount_data = state["refcount_data"] stable_abi_data = state["stable_abi_data"] + threadsafety_data = state["threadsafety_data"] for node in doctree.findall(addnodes.desc_content): par = node.parent if par["domain"] != "c": @@ -126,6 +169,12 @@ def add_annotations(app: Sphinx, doctree: nodes.document) -> None: name = par[0]["ids"][0].removeprefix("c.") objtype = par["objtype"] + # Thread safety annotation — inserted first so it appears last (bottom-most) + # among all annotations. + if entry := threadsafety_data.get(name): + annotation = _threadsafety_annotation(entry.level) + node.insert(0, annotation) + # Stable ABI annotation. if record := stable_abi_data.get(name): if ROLE_TO_OBJECT_TYPE[record.role] != objtype: @@ -154,7 +203,10 @@ def add_annotations(app: Sphinx, doctree: nodes.document) -> None: node.insert(0, annotation) -def _stable_abi_annotation(record: StableABIEntry) -> nodes.emphasis: +def _stable_abi_annotation( + record: StableABIEntry, + is_corresponding_slot: bool = False, +) -> nodes.emphasis: """Create the Stable ABI annotation. These have two forms: @@ -168,9 +220,28 @@ def _stable_abi_annotation(record: StableABIEntry) -> nodes.emphasis: ... all of which can have "since version X.Y" appended. """ stable_added = record.added - message = sphinx_gettext("Part of the") - message = message.center(len(message) + 2) - emph_node = nodes.emphasis(message, message, classes=["stableabi"]) + emph_node = nodes.emphasis('', '', classes=["stableabi"]) + if is_corresponding_slot: + # See "Type slot annotations" in add_annotations + ref_node = addnodes.pending_xref( + "slot ID", + refdomain="c", + reftarget="PyType_Slot", + reftype="type", + refexplicit="True", + ) + ref_node += nodes.Text(sphinx_gettext("slot ID")) + + message = sphinx_gettext("The corresponding") + emph_node += nodes.Text(" " + message + " ") + emph_node += ref_node + emph_node += nodes.Text(" ") + emph_node += nodes.literal(record.name, record.name) + message = sphinx_gettext("is part of the") + emph_node += nodes.Text(" " + message + " ") + else: + message = sphinx_gettext("Part of the") + emph_node += nodes.Text(" " + message + " ") ref_node = addnodes.pending_xref( "Stable ABI", refdomain="std", @@ -178,18 +249,17 @@ def _stable_abi_annotation(record: StableABIEntry) -> nodes.emphasis: reftype="ref", refexplicit="False", ) - struct_abi_kind = record.struct_abi_kind - if struct_abi_kind in {"opaque", "members"}: - ref_node += nodes.Text(sphinx_gettext("Limited API")) - else: - ref_node += nodes.Text(sphinx_gettext("Stable ABI")) + ref_node += nodes.Text(sphinx_gettext("Stable ABI")) emph_node += ref_node + struct_abi_kind = record.struct_abi_kind if struct_abi_kind == "opaque": emph_node += nodes.Text(" " + sphinx_gettext("(as an opaque struct)")) elif struct_abi_kind == "full-abi": emph_node += nodes.Text( " " + sphinx_gettext("(including all members)") ) + elif struct_abi_kind in {"members", "abi3t-opaque"}: + emph_node += nodes.Text(" " + sphinx_gettext("(see below)")) if record.ifdef_note: emph_node += nodes.Text(f" {record.ifdef_note}") if stable_added == "3.2": @@ -200,11 +270,7 @@ def _stable_abi_annotation(record: StableABIEntry) -> nodes.emphasis: " " + sphinx_gettext("since version %s") % stable_added ) emph_node += nodes.Text(".") - if struct_abi_kind == "members": - msg = " " + sphinx_gettext( - "(Only some members are part of the stable ABI.)" - ) - emph_node += nodes.Text(msg) + return emph_node @@ -234,6 +300,48 @@ def _unstable_api_annotation() -> nodes.admonition: ) +def _threadsafety_annotation(level: str) -> nodes.emphasis: + match level: + case "incompatible": + display = sphinx_gettext("Not safe to call from multiple threads") + reftarget = "threadsafety-level-incompatible" + case "compatible": + display = sphinx_gettext( + "Safe to call from multiple threads" + " with external synchronization only" + ) + reftarget = "threadsafety-level-compatible" + case "distinct": + display = sphinx_gettext( + "Safe to call without external synchronization" + " on distinct objects" + ) + reftarget = "threadsafety-level-distinct" + case "shared": + display = sphinx_gettext( + "Safe for concurrent use on the same object" + ) + reftarget = "threadsafety-level-shared" + case "atomic": + display = sphinx_gettext("Atomic") + reftarget = "threadsafety-level-atomic" + case _: + raise AssertionError(f"Unknown thread safety level {level!r}") + ref_node = addnodes.pending_xref( + display, + nodes.Text(display), + refdomain="std", + reftarget=reftarget, + reftype="ref", + refexplicit="True", + ) + prefix = " " + sphinx_gettext("Thread safety:") + " " + classes = ["threadsafety", f"threadsafety-{level}"] + return nodes.emphasis( + "", prefix, ref_node, nodes.Text("."), classes=classes + ) + + def _return_value_annotation(result_refs: int | None) -> nodes.emphasis: classes = ["refcount"] if result_refs is None: @@ -265,6 +373,78 @@ def run(self) -> list[nodes.Node]: return [node] +class VersionHexCheatsheet(SphinxDirective): + """Show results of Py_PACK_VERSION(3, x) for a few relevant Python versions + + This is useful for defining version before Python.h is included. + It should auto-update with the version being documented, so it must be an + extension. + """ + + has_content = False + required_arguments = 0 + optional_arguments = 0 + final_argument_whitespace = True + + def run(self) -> list[nodes.Node]: + content = [ + ".. code-block:: c", + "", + ] + current_minor = int(self.config.version.removeprefix('3.')) + for minor in range(current_minor - 5, current_minor + 1): + value = (3 << 24) | (minor << 16) + content.append(f' {value:#x} /* Py_PACK_VERSION(3.{minor}) */') + node = nodes.paragraph() + self.state.nested_parse(StringList(content), 0, node) + return [node] + + +class CorrespondingTypeSlot(SphinxDirective): + """Type slot annotations + + Docs for these are with the corresponding field, for example, + "Py_tp_repr" is documented under "PyTypeObject.tp_repr", with + only a stable ABI note mentioning "Py_tp_repr" (and linking to + docs on how this works). + + If there is no corresponding field, these should be documented as normal + macros. + """ + + has_content = False + + required_arguments = 1 + optional_arguments = 0 + + def run(self) -> list[nodes.Node]: + name = self.arguments[0] + state = self.env.domaindata["c_annotations"] + stable_abi_data = state["stable_abi_data"] + + try: + record = stable_abi_data[name] + except LookupError as err: + raise LookupError( + f"{name} is not part of stable ABI. " + + "Document it as `c:macro::` rather than " + + "`corresponding-type-slot::`." + ) from err + + annotation = _stable_abi_annotation(record, is_corresponding_slot=True) + + node = nodes.paragraph() + content = [ + ".. c:namespace:: NULL", + "", + ".. c:macro:: " + name, + " :no-typesetting:", + ] + self.state.nested_parse(StringList(content), 0, node) + node.insert(0, annotation) + return [node] + + def init_annotations(app: Sphinx) -> None: # Using domaindata is a bit hack-ish, # but allows storing state without a global variable or closure. @@ -275,12 +455,18 @@ def init_annotations(app: Sphinx) -> None: state["stable_abi_data"] = read_stable_abi_data( Path(app.srcdir, app.config.stable_abi_file) ) + state["threadsafety_data"] = read_threadsafety_data( + Path(app.srcdir, app.config.threadsafety_file) + ) def setup(app: Sphinx) -> ExtensionMetadata: app.add_config_value("refcount_file", "", "env", types={str}) app.add_config_value("stable_abi_file", "", "env", types={str}) + app.add_config_value("threadsafety_file", "", "env", types={str}) app.add_directive("limited-api-list", LimitedAPIList) + app.add_directive("version-hex-cheatsheet", VersionHexCheatsheet) + app.add_directive("corresponding-type-slot", CorrespondingTypeSlot) app.connect("builder-inited", init_annotations) app.connect("doctree-read", add_annotations) diff --git a/Doc/tools/extensions/changes.py b/Doc/tools/extensions/changes.py index 8de5e7f78c6..02dc51b3a76 100644 --- a/Doc/tools/extensions/changes.py +++ b/Doc/tools/extensions/changes.py @@ -2,8 +2,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING +import re +from docutils import nodes +from sphinx import addnodes from sphinx.domains.changeset import ( VersionChange, versionlabel_classes, @@ -11,6 +13,7 @@ ) from sphinx.locale import _ as sphinx_gettext +TYPE_CHECKING = False if TYPE_CHECKING: from docutils.nodes import Node from sphinx.application import Sphinx @@ -73,6 +76,76 @@ def run(self) -> list[Node]: versionlabel_classes[self.name] = "" +class SoftDeprecated(PyVersionChange): + """Directive for soft deprecations that auto-links to the glossary term. + + Usage:: + + .. soft-deprecated:: 3.15 + + Use :func:`new_thing` instead. + + Renders as: "Soft deprecated since version 3.15: Use new_thing() instead." + with "Soft deprecated" linking to the glossary definition. + """ + + _TERM_RE = re.compile(r":term:`([^`]+)`") + + def run(self) -> list[Node]: + versionlabels[self.name] = sphinx_gettext( + ":term:`Soft deprecated` since version %s" + ) + versionlabel_classes[self.name] = "soft-deprecated" + try: + result = super().run() + finally: + versionlabels[self.name] = "" + versionlabel_classes[self.name] = "" + + for node in result: + # Add "versionchanged" class so existing theme CSS applies + node["classes"] = node.get("classes", []) + ["versionchanged"] + # Replace the plain-text "Soft deprecated" with a glossary reference + for inline in node.findall(nodes.inline): + if "versionmodified" in inline.get("classes", []): + self._add_glossary_link(inline) + + return result + + @classmethod + def _add_glossary_link(cls, inline: nodes.inline) -> None: + """Replace :term:`soft deprecated` text with a cross-reference to the + 'Soft deprecated' glossary entry.""" + for child in inline.children: + if not isinstance(child, nodes.Text): + continue + + text = str(child) + match = cls._TERM_RE.search(text) + if match is None: + continue + + ref = addnodes.pending_xref( + "", + nodes.Text(match.group(1)), + refdomain="std", + reftype="term", + reftarget="soft deprecated", + refwarn=True, + ) + + start, end = match.span() + new_nodes: list[nodes.Node] = [] + if start > 0: + new_nodes.append(nodes.Text(text[:start])) + new_nodes.append(ref) + if end < len(text): + new_nodes.append(nodes.Text(text[end:])) + + child.parent.replace(child, new_nodes) + break + + def setup(app: Sphinx) -> ExtensionMetadata: # Override Sphinx's directives with support for 'next' app.add_directive("versionadded", PyVersionChange, override=True) @@ -83,6 +156,9 @@ def setup(app: Sphinx) -> ExtensionMetadata: # Register the ``.. deprecated-removed::`` directive app.add_directive("deprecated-removed", DeprecatedRemoved) + # Register the ``.. soft-deprecated::`` directive + app.add_directive("soft-deprecated", SoftDeprecated) + return { "version": "1.0", "parallel_read_safe": True, diff --git a/Doc/tools/extensions/glossary_search.py b/Doc/tools/extensions/glossary_search.py index 502b6cd95bc..bcda4b7b435 100644 --- a/Doc/tools/extensions/glossary_search.py +++ b/Doc/tools/extensions/glossary_search.py @@ -38,7 +38,7 @@ def process_glossary_nodes( rendered = app.builder.render_partial(definition) terms[term.lower()] = { 'title': term, - 'body': rendered['html_body'], + 'body': rendered['fragment'], } diff --git a/Doc/tools/extensions/grammar_snippet.py b/Doc/tools/extensions/grammar_snippet.py index 1e059f111e4..8078b7ebeb8 100644 --- a/Doc/tools/extensions/grammar_snippet.py +++ b/Doc/tools/extensions/grammar_snippet.py @@ -191,7 +191,7 @@ class GrammarSnippetDirective(GrammarSnippetBase): into something similar to Sphinx productionlist, but better suited for our needs: - Instead of `::=`, use a colon, as in `Grammar/python.gram` - - Show the listing almost as is, with no auto-aligment. + - Show the listing almost as is, with no auto-alignment. The only special character is the backtick, which marks tokens. Unlike Sphinx's productionlist, this directive supports options. diff --git a/Doc/tools/extensions/lexers/asdl_lexer.py b/Doc/tools/extensions/lexers/asdl_lexer.py index 3a74174a1f7..548ce1271a1 100644 --- a/Doc/tools/extensions/lexers/asdl_lexer.py +++ b/Doc/tools/extensions/lexers/asdl_lexer.py @@ -22,7 +22,7 @@ class ASDLLexer(RegexLexer): bygroups(Keyword, Text, Name.Tag), ), ( - r"(\w+)(\*\s|\?\s|\s)(\w+)", + r"(\w+)([\?\*]*\s)(\w+)", bygroups(Name.Builtin.Pseudo, Operator, Name), ), # Keep in line with ``builtin_types`` from Parser/asdl.py. diff --git a/Doc/tools/extensions/profiling_trace.py b/Doc/tools/extensions/profiling_trace.py new file mode 100644 index 00000000000..7185ef351dd --- /dev/null +++ b/Doc/tools/extensions/profiling_trace.py @@ -0,0 +1,166 @@ +""" +Sphinx extension to generate profiler trace data during docs build. + +This extension executes a demo Python program with sys.settrace() to capture +the execution trace and injects it into the profiling visualization JS file. +""" + +import json +import re +import sys +from io import StringIO +from pathlib import Path + +from sphinx.errors import ExtensionError + +DEMO_SOURCE = """\ +def add(a, b): + return a + b + +def multiply(x, y): + result = 0 + for i in range(y): + result = add(result, x) + return result + +def calculate(a, b): + sum_val = add(a, b) + product = multiply(a, b) + return sum_val + product + +def main(): + result = calculate(3, 4) + print(f"Result: {result}") + +main() +""" + +PLACEHOLDER = "/* PROFILING_TRACE_DATA */null" + + +def generate_trace(source: str) -> list[dict]: + """ + Execute the source code with tracing enabled and capture execution events. + """ + trace_events = [] + timestamp = [0] + timestamp_step = 50 + tracing_active = [False] + pending_line = [None] + + def tracer(frame, event, arg): + if frame.f_code.co_filename != '': + return tracer + + func_name = frame.f_code.co_name + lineno = frame.f_lineno + + if event == 'line' and not tracing_active[0]: + pending_line[0] = {'type': 'line', 'line': lineno} + return tracer + + # Start tracing only once main() is called + if event == 'call' and func_name == 'main': + tracing_active[0] = True + # Emit the buffered line event (the main() call line) at ts=0 + if pending_line[0]: + pending_line[0]['ts'] = 0 + trace_events.append(pending_line[0]) + pending_line[0] = None + timestamp[0] = timestamp_step + + # Skip events until we've entered main() + if not tracing_active[0]: + return tracer + + if event == 'call': + trace_events.append({ + 'type': 'call', + 'func': func_name, + 'line': lineno, + 'ts': timestamp[0], + }) + elif event == 'line': + trace_events.append({ + 'type': 'line', + 'line': lineno, + 'ts': timestamp[0], + }) + elif event == 'return': + try: + value = arg if arg is None else repr(arg) + except Exception: + value = '' + trace_events.append({ + 'type': 'return', + 'func': func_name, + 'ts': timestamp[0], + 'value': value, + }) + + if func_name == 'main': + tracing_active[0] = False + + timestamp[0] += timestamp_step + return tracer + + # Suppress print output during tracing + old_stdout = sys.stdout + sys.stdout = StringIO() + + old_trace = sys.gettrace() + sys.settrace(tracer) + try: + code = compile(source, '', 'exec') + exec(code, {'__name__': '__main__'}) + finally: + sys.settrace(old_trace) + sys.stdout = old_stdout + + return trace_events + + +def inject_trace(app, exception): + if exception: + return + + js_path = ( + Path(app.outdir) / '_static' / 'profiling-sampling-visualization.js' + ) + + if not js_path.exists(): + return + + trace = generate_trace(DEMO_SOURCE) + + demo_data = {'source': DEMO_SOURCE.rstrip(), 'trace': trace, 'samples': []} + + demo_json = json.dumps(demo_data, indent=2) + content = js_path.read_text(encoding='utf-8') + + pattern = r"(const DEMO_SIMPLE\s*=\s*/\* PROFILING_TRACE_DATA \*/)[^;]+;" + + if re.search(pattern, content): + content = re.sub( + pattern, lambda m: f"{m.group(1)} {demo_json};", content + ) + js_path.write_text(content, encoding='utf-8') + print( + f"profiling_trace: Injected {len(trace)} trace events into {js_path.name}" + ) + else: + raise ExtensionError( + f"profiling_trace: Placeholder pattern not found in {js_path.name}" + ) + + +def setup(app): + app.connect('build-finished', inject_trace) + app.add_js_file('profiling-sampling-visualization.js') + app.add_css_file('profiling-sampling-visualization.css') + + return { + 'version': '1.0', + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/Doc/tools/extensions/pydoc_topics.py b/Doc/tools/extensions/pydoc_topics.py index 01efbba6283..a65d77433b2 100644 --- a/Doc/tools/extensions/pydoc_topics.py +++ b/Doc/tools/extensions/pydoc_topics.py @@ -109,6 +109,7 @@ class PydocTopicsBuilder(TextBuilder): def init(self) -> None: super().init() self.topics: dict[str, str] = {} + self.module_docs: dict[str, str] = {} def get_outdated_docs(self) -> str: # Return a string describing what an update build will build. @@ -130,6 +131,15 @@ def write_documents(self, _docnames: Set[str]) -> None: continue doc_labels.setdefault(docname, []).append((topic_label, label_id)) + py_domain = env.domains['py'] + for module_name, module_info in py_domain.data['modules'].items(): + docname = module_info[0] + if docname.startswith('library/'): + doc_file = docname.replace('library/', '') + self.module_docs[module_name] = ( + f"{doc_file}#module-{module_name}" + ) + for docname, label_ids in status_iterator( doc_labels.items(), "building topics... ", @@ -161,6 +171,22 @@ def finish(self) -> None: """ self.outdir.joinpath("topics.py").write_text(topics, encoding="utf-8") + module_docs_repr = "\n".join( + f" '{module}': '{doc_file}'," + for module, doc_file in sorted(self.module_docs.items()) + ) + module_docs = f"""\ +# Autogenerated by Sphinx on {asctime()} +# as part of the release process. + +module_docs = {{ +{module_docs_repr} +}} +""" + self.outdir.joinpath("module_docs.py").write_text( + module_docs, encoding="utf-8" + ) + def _display_labels(item: tuple[str, Sequence[tuple[str, str]]]) -> str: _docname, label_ids = item diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index f5451adb37b..f9bf273e762 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -24,14 +24,6 @@ # Used in conf.py and updated here by python/release-tools/run_release.py SOURCE_URI = 'https://github.com/python/cpython/tree/main/%s' -# monkey-patch reST parser to disable alphabetic and roman enumerated lists -from docutils.parsers.rst.states import Body -Body.enum.converters['loweralpha'] = \ - Body.enum.converters['upperalpha'] = \ - Body.enum.converters['lowerroman'] = \ - Body.enum.converters['upperroman'] = lambda x: None - - class PyAwaitableMixin(object): def handle_signature(self, sig, signode): ret = super(PyAwaitableMixin, self).handle_signature(sig, signode) diff --git a/Doc/tools/removed-ids.txt b/Doc/tools/removed-ids.txt new file mode 100644 index 00000000000..5e3ef2efe27 --- /dev/null +++ b/Doc/tools/removed-ids.txt @@ -0,0 +1,7 @@ +# HTML IDs excluded from the check-html-ids.py check. + +# Remove from here in 3.16 +c-api/allocation.html: deprecated-aliases +c-api/file.html: deprecated-api + +library/asyncio-task.html: terminating-a-task-group diff --git a/Doc/tools/templates/customsourcelink.html b/Doc/tools/templates/customsourcelink.html index 8e271bca1e0..8feeed2fee3 100644 --- a/Doc/tools/templates/customsourcelink.html +++ b/Doc/tools/templates/customsourcelink.html @@ -1,13 +1,34 @@ {%- if show_source and has_source and sourcename %} +
{%- endif %} diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index 47a57eb111b..c78c650b1cb 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -31,8 +31,7 @@

{% trans %}Download Python {{ dl_version }} documentation{% endtrans %}

{% if last_updated %}

{% trans %}Last updated on: {{ last_updated }}.{% endtrans %}

{% endif %} -

{% trans %}To download an archive containing all the documents for this version of -Python in one of various formats, follow one of links in this table.{% endtrans %}

+

{% trans %}Download an archive containing all the documentation for this version of Python:{% endtrans %}

@@ -40,50 +39,41 @@

{% trans %}Download Python {{ dl_version }} documentation{% endtrans %}

- - - - - - - + + - - + + - - + + - +
{% trans %}Packed as .zip{% endtrans %} {% trans %}Packed as .tar.bz2{% endtrans %}
{% trans %}PDF{% endtrans %}{% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %}
{% trans %}HTML{% endtrans %}{% trans download_size="13" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans download_size="8" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans %}Download{% endtrans %}{% trans %}Download{% endtrans %}
{% trans %}Plain text{% endtrans %}{% trans download_size="4" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans download_size="3" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans %}Download{% endtrans %}{% trans %}Download{% endtrans %}
{% trans %}Texinfo{% endtrans %}{% trans download_size="9" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans download_size="7" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans %}Download{% endtrans %}{% trans %}Download{% endtrans %}
{% trans %}EPUB{% endtrans %}{% trans download_size="6" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans %}Download{% endtrans %}
-

{% trans %}These archives contain all the content in the documentation.{% endtrans %}

- - -

{% trans %}Unpacking{% endtrans %}

- -

{% trans %}Unix users should download the .tar.bz2 archives; these are bzipped tar -archives and can be handled in the usual way using tar and the bzip2 -program. The Info-ZIP unzip program can be -used to handle the ZIP archives if desired. The .tar.bz2 archives provide the -best compression and fastest download times.{% endtrans %}

- -

{% trans %}Windows users can use the ZIP archives since those are customary on that -platform. These are created on Unix using the Info-ZIP zip program.{% endtrans %}

+

{% trans %} +We no longer provide pre-built PDFs of the documentation. +To build a PDF archive, follow the instructions in the +Developer's Guide +and run make dist-pdf in the Doc/ directory of a copy of the CPython repository. +{% endtrans %}

+

{% trans %} +See the directory listing +for file sizes.{% endtrans %}

{% trans %}Problems{% endtrans %}

- -

{% trans %}If you have comments or suggestions for the Python documentation, please send -email to docs@python.org.{% endtrans %}

+{% set bugs = pathto('bugs') %} +

{% trans bugs = bugs %}Open an issue +if you have comments or suggestions for the Python documentation.{% endtrans %}

{% endblock %} diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html index 0fdbe2a5801..699e518801c 100644 --- a/Doc/tools/templates/dummy.html +++ b/Doc/tools/templates/dummy.html @@ -27,8 +27,9 @@ In extensions/changes.py: -{% trans %}Deprecated since version {deprecated}, will be removed in version {removed}{% endtrans %} -{% trans %}Deprecated since version {deprecated}, removed in version {removed}{% endtrans %} +{% trans %}Deprecated since version %s, will be removed in version %s{% endtrans %} +{% trans %}Deprecated since version %s, removed in version %s{% endtrans %} +{% trans %}:term:`Soft deprecated` since version %s{% endtrans %} In docsbuild-scripts, when rewriting indexsidebar.html with actual versions: diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index 5020c428741..2539edb51ba 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -14,8 +14,7 @@ There are two variants of the interactive :term:`REPL`. The classic basic interpreter is supported on all platforms with minimal line control capabilities. -On Windows, or Unix-like systems with :mod:`curses` support, -a new interactive shell is used by default since Python 3.13. +Since Python 3.13, a new interactive shell is used by default. This one supports color, multiline editing, history browsing, and paste mode. To disable color, see :ref:`using-on-controlling-color` for details. Function keys provide some additional functionality. diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 9ab003d5cd3..4f7da5253f7 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -420,7 +420,7 @@ of the class:: 'Buddy' As discussed in :ref:`tut-object`, shared data can have possibly surprising -effects with involving :term:`mutable` objects such as lists and dictionaries. +effects involving :term:`mutable` objects such as lists and dictionaries. For example, the *tricks* list in the following code should not be used as a class variable because just a single list would be shared by all *Dog* instances:: @@ -929,6 +929,25 @@ Examples:: >>> list(data[i] for i in range(len(data)-1, -1, -1)) ['f', 'l', 'o', 'g'] + >>> x = [[1,2,3], [], [4, 5]] + >>> g = (*i for i in x) + >>> list(g) + [1, 2, 3, 4, 5] + +In most cases, generator expressions must be wrapped in parentheses. As a +special case, however, when provided as the sole argument to a function (as in +the examples involving ``sum``, ``set``, ``max``, and ``list`` above), the +generator expression does not need to be wrapped in an additional set of +parentheses. That is to say, the following two pieces of code are semantically +equivalent:: + + >>> f(x for x in y) + >>> f((x for x in y)) + +as are the following:: + + >>> f(*x for x in y) + >>> f((*x for x in y)) .. rubric:: Footnotes diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 5ec8789f98c..8bac8df4368 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -155,8 +155,8 @@ that takes an iterable is :func:`sum`:: 6 Later we will see more functions that return iterables and take iterables as -arguments. In chapter :ref:`tut-structures`, we will discuss in more detail about -:func:`list`. +arguments. In chapter :ref:`tut-structures`, we will discuss :func:`list` in more +detail. .. _tut-break: @@ -441,7 +441,7 @@ Several other key features of this statement: ``False`` and ``None`` are compared by identity. - Patterns may use named constants. These must be dotted names - to prevent them from being interpreted as capture variable:: + to prevent them from being interpreted as capture variables:: from enum import Enum class Color(Enum): @@ -1039,31 +1039,28 @@ blank, visually separating the summary from the rest of the description. The following lines should be one or more paragraphs describing the object's calling conventions, its side effects, etc. -The Python parser does not strip indentation from multi-line string literals in -Python, so tools that process documentation have to strip indentation if -desired. This is done using the following convention. The first non-blank line -*after* the first line of the string determines the amount of indentation for -the entire documentation string. (We can't use the first line since it is -generally adjacent to the string's opening quotes so its indentation is not -apparent in the string literal.) Whitespace "equivalent" to this indentation is -then stripped from the start of all lines of the string. Lines that are -indented less should not occur, but if they occur all their leading whitespace -should be stripped. Equivalence of whitespace should be tested after expansion -of tabs (to 8 spaces, normally). +The Python parser strips indentation from multi-line string literals when they +serve as module, class, or function docstrings. Here is an example of a multi-line docstring:: >>> def my_function(): ... """Do nothing, but document it. ... - ... No, really, it doesn't do anything. + ... No, really, it doesn't do anything: + ... + ... >>> my_function() + ... >>> ... """ ... pass ... >>> print(my_function.__doc__) Do nothing, but document it. - No, really, it doesn't do anything. + No, really, it doesn't do anything: + + >>> my_function() + >>> .. _tut-annotations: @@ -1071,7 +1068,6 @@ Here is an example of a multi-line docstring:: Function Annotations -------------------- -.. sectionauthor:: Zachary Ware .. index:: pair: function; annotations single: ->; function annotations @@ -1105,12 +1101,11 @@ value annotated:: Intermezzo: Coding Style ======================== -.. sectionauthor:: Georg Brandl .. index:: pair: coding; style Now that you are about to write longer, more complex pieces of Python, it is a good time to talk about *coding style*. Most languages can be written (or more -concise, *formatted*) in different styles; some are more readable than others. +concisely, *formatted*) in different styles; some are more readable than others. Making it easy for others to read your code is always a good idea, and adopting a nice coding style helps tremendously for that. diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index db8a066b369..276e31a3056 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -12,24 +12,23 @@ and adds some new things as well. More on Lists ============= -The list data type has some more methods. Here are all of the methods of list -objects: +The :ref:`list ` data type has some more methods. Here are all +of the methods of list objects: - -.. method:: list.append(x) +.. method:: list.append(value, /) :noindex: Add an item to the end of the list. Similar to ``a[len(a):] = [x]``. -.. method:: list.extend(iterable) +.. method:: list.extend(iterable, /) :noindex: Extend the list by appending all the items from the iterable. Similar to ``a[len(a):] = iterable``. -.. method:: list.insert(i, x) +.. method:: list.insert(index, value, /) :noindex: Insert an item at a given position. The first argument is the index of the @@ -37,14 +36,14 @@ objects: the list, and ``a.insert(len(a), x)`` is equivalent to ``a.append(x)``. -.. method:: list.remove(x) +.. method:: list.remove(value, /) :noindex: - Remove the first item from the list whose value is equal to *x*. It raises a + Remove the first item from the list whose value is equal to *value*. It raises a :exc:`ValueError` if there is no such item. -.. method:: list.pop([i]) +.. method:: list.pop(index=-1, /) :noindex: Remove the item at the given position in the list, and return it. If no index @@ -59,10 +58,10 @@ objects: Remove all items from the list. Similar to ``del a[:]``. -.. method:: list.index(x[, start[, end]]) +.. method:: list.index(value[, start[, stop]]) :noindex: - Return zero-based index of the first occurrence of *x* in the list. + Return zero-based index of the first occurrence of *value* in the list. Raises a :exc:`ValueError` if there is no such item. The optional arguments *start* and *end* are interpreted as in the slice @@ -71,10 +70,10 @@ objects: sequence rather than the *start* argument. -.. method:: list.count(x) +.. method:: list.count(value, /) :noindex: - Return the number of times *x* appears in the list. + Return the number of times *value* appears in the list. .. method:: list.sort(*, key=None, reverse=False) @@ -137,9 +136,6 @@ comparison. Using Lists as Stacks --------------------- -.. sectionauthor:: Ka-Ping Yee - - The list methods make it very easy to use a list as a stack, where the last element added is the first element retrieved ("last-in, first-out"). To add an item to the top of the stack, use :meth:`~list.append`. To retrieve an item from the @@ -167,8 +163,6 @@ top of the stack, use :meth:`~list.pop` without an explicit index. For example: Using Lists as Queues --------------------- -.. sectionauthor:: Ka-Ping Yee - It is also possible to use a list as a queue, where the first element added is the first element retrieved ("first-in, first-out"); however, lists are not efficient for this purpose. While appends and pops from the end of list are @@ -334,6 +328,47 @@ The :func:`zip` function would do a great job for this use case:: See :ref:`tut-unpacking-arguments` for details on the asterisk in this line. +Unpacking in Lists and List Comprehensions +------------------------------------------ + +The section on :ref:`tut-unpacking-arguments` describes the use of ``*`` to +"unpack" the elements of an iterable object, providing each one separately as +an argument to a function. Unpacking can also be used in other contexts, for +example, when creating lists. When specifying elements of a list, prefixing an +expression by a ``*`` will unpack the result of that expression, adding each of +its elements to the list we're creating:: + + >>> x = [1, 2, 3] + >>> [0, *x, 4, 5, 6] + [0, 1, 2, 3, 4, 5, 6] + +This only works if the expression following the ``*`` evaluates to an iterable +object; trying to unpack a non-iterable object will raise an exception:: + + >>> x = 1 + >>> [0, *x, 2, 3, 4] + Traceback (most recent call last): + File "", line 1, in + [0, *x, 2, 3, 4] + TypeError: Value after * must be an iterable, not int + +Unpacking can also be used in list comprehensions, as a way to build a new list +representing the concatenation of an arbitrary number of iterables:: + + >>> x = [[1, 2, 3], [4, 5, 6], [], [7], [8, 9]] + >>> [*element for element in x] + [1, 2, 3, 4, 5, 6, 7, 8, 9] + +Note that the effect is that each element from ``x`` is unpacked. This works +for arbitrary iterable objects, not just lists:: + + >>> x = [[1, 2, 3], 'cat', {'spam': 'eggs'}] + >>> [*element for element in x] + [1, 2, 3, 'c', 'a', 't', 'spam'] + +But if the objects in ``x`` are not iterable, this expression would again raise +an exception. + .. _tut-del: The :keyword:`!del` statement @@ -395,7 +430,10 @@ A tuple consists of a number of values separated by commas, for instance:: >>> v = ([1, 2, 3], [3, 2, 1]) >>> v ([1, 2, 3], [3, 2, 1]) - + >>> # they support unpacking just like lists: + >>> x = [1, 2, 3] + >>> 0, *x, 4 + (0, 1, 2, 3, 4) As you see, on output tuples are always enclosed in parentheses, so that nested tuples are interpreted correctly; they may be input with or without surrounding @@ -445,15 +483,19 @@ packing and sequence unpacking. Sets ==== -Python also includes a data type for *sets*. A set is an unordered collection -with no duplicate elements. Basic uses include membership testing and -eliminating duplicate entries. Set objects also support mathematical operations -like union, intersection, difference, and symmetric difference. +Python also includes a data type for :ref:`sets `. A set is +an unordered collection with no duplicate elements. Basic uses include +membership testing and eliminating duplicate entries. Set objects also +support mathematical operations like union, intersection, difference, and +symmetric difference. Curly braces or the :func:`set` function can be used to create sets. Note: to create an empty set you have to use ``set()``, not ``{}``; the latter creates an empty dictionary, a data structure that we discuss in the next section. +Because sets are unordered, iterating over them or printing them can +produce the elements in a different order than you expect. + Here is a brief demonstration:: >>> basket = {'apple', 'orange', 'apple', 'pear', 'orange', 'banana'} @@ -480,12 +522,16 @@ Here is a brief demonstration:: {'r', 'd', 'b', 'm', 'z', 'l'} Similarly to :ref:`list comprehensions `, set comprehensions -are also supported:: +are also supported, including comprehensions with unpacking:: >>> a = {x for x in 'abracadabra' if x not in 'abc'} >>> a {'r', 'd'} + >>> fruits = [{'apple', 'avocado', 'apricot'}, {'banana', 'blueberry'}] + >>> {*fruit for fruit in fruits} + {'blueberry', 'banana', 'avocado', 'apple', 'apricot'} + .. _tut-dictionaries: @@ -512,8 +558,12 @@ dictionary; this is also the way dictionaries are written on output. The main operations on a dictionary are storing a value with some key and extracting the value given the key. It is also possible to delete a key:value pair with ``del``. If you store using a key that is already in use, the old -value associated with that key is forgotten. It is an error to extract a value -using a non-existent key. +value associated with that key is forgotten. + +Extracting a value for a non-existent key by subscripting (``d[key]``) raises a +:exc:`KeyError`. To avoid getting this error when trying to access a possibly +non-existent key, use the :meth:`~dict.get` method instead, which returns +``None`` (or a specified default value) if the key is not in the dictionary. Performing ``list(d)`` on a dictionary returns a list of all the keys used in the dictionary, in insertion order (if you want it sorted, just use @@ -528,6 +578,12 @@ Here is a small example using a dictionary:: {'jack': 4098, 'sape': 4139, 'guido': 4127} >>> tel['jack'] 4098 + >>> tel['irv'] + Traceback (most recent call last): + File "", line 1, in + KeyError: 'irv' + >>> print(tel.get('irv')) + None >>> del tel['sape'] >>> tel['irv'] = 4127 >>> tel @@ -553,6 +609,18 @@ arbitrary key and value expressions:: >>> {x: x**2 for x in (2, 4, 6)} {2: 4, 4: 16, 6: 36} +And dictionary unpacking (via ``**``) can be used to merge multiple +dictionaries:: + + >>> odds = {i: i**2 for i in (1, 3, 5)} + >>> evens = {i: i**2 for i in (2, 4, 6)} + >>> {**odds, **evens} + {1: 1, 3: 9, 5: 25, 2: 4, 4: 16, 6: 36} + + >>> all_values = [odds, evens, {0: 0}] + >>> {**i for i in all_values} + {1: 1, 3: 9, 5: 25, 2: 4, 4: 16, 6: 36, 0: 0} + When the keys are simple strings, it is sometimes easier to specify pairs using keyword arguments:: diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 1c20fa2f0b6..3c6edf2c479 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -121,9 +121,9 @@ A :keyword:`try` statement may have more than one *except clause*, to specify handlers for different exceptions. At most one handler will be executed. Handlers only handle exceptions that occur in the corresponding *try clause*, not in other handlers of the same :keyword:`!try` statement. An *except clause* -may name multiple exceptions as a parenthesized tuple, for example:: +may name multiple exceptions, for example:: - ... except (RuntimeError, TypeError, NameError): + ... except RuntimeError, TypeError, NameError: ... pass A class in an :keyword:`except` clause matches exceptions which are instances of the @@ -549,9 +549,9 @@ caught like any other exception. :: >>> try: ... f() ... except Exception as e: - ... print(f'caught {type(e)}: e') + ... print(f'caught {type(e)}: {e}') ... - caught : e + caught : there were problems (2 sub-exceptions) >>> By using ``except*`` instead of ``except``, we can selectively diff --git a/Doc/tutorial/floatingpoint.rst b/Doc/tutorial/floatingpoint.rst index dfe2d1d3a83..37e23ba1cd0 100644 --- a/Doc/tutorial/floatingpoint.rst +++ b/Doc/tutorial/floatingpoint.rst @@ -9,10 +9,6 @@ Floating-Point Arithmetic: Issues and Limitations ************************************************** -.. sectionauthor:: Tim Peters -.. sectionauthor:: Raymond Hettinger - - Floating-point numbers are represented in computer hardware as base 2 (binary) fractions. For example, the **decimal** fraction ``0.625`` has value 6/10 + 2/100 + 5/1000, and in the same way the **binary** fraction ``0.101`` diff --git a/Doc/tutorial/index.rst b/Doc/tutorial/index.rst index d0bf77dc40d..20fe161be4a 100644 --- a/Doc/tutorial/index.rst +++ b/Doc/tutorial/index.rst @@ -15,7 +15,7 @@ together with its interpreted nature, make it an ideal language for scripting and rapid application development in many areas on most platforms. The Python interpreter and the extensive standard library are freely available -in source or binary form for all major platforms from the Python web site, +in source or binary form for all major platforms from the Python website, https://www.python.org/, and may be freely distributed. The same site also contains distributions of and pointers to many free third party Python modules, programs and tools, and additional documentation. diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index cd526071424..72cac1c1e90 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -34,13 +34,13 @@ status. If that doesn't work, you can exit the interpreter by typing the following command: ``quit()``. The interpreter's line-editing features include interactive editing, history -substitution and code completion on systems that support the `GNU Readline -`_ library. +substitution and code completion on most systems. Perhaps the quickest check to see whether command line editing is supported is -typing :kbd:`Control-P` to the first Python prompt you get. If it beeps, you -have command line editing; see Appendix :ref:`tut-interacting` for an -introduction to the keys. If nothing appears to happen, or if ``^P`` is -echoed, command line editing isn't available; you'll only be able to use +typing a word in on the Python prompt, then pressing Left arrow (or :kbd:`Control-b`). +If the cursor moves, you have command line editing; see Appendix +:ref:`tut-interacting` for an introduction to the keys. +If nothing appears to happen, or if a sequence like ``^[[D`` or ``^B`` appears, +command line editing isn't available; you'll only be able to use backspace to remove characters from the current line. The interpreter operates somewhat like the Unix shell: when called with standard diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index fb491149793..7778e37a9ad 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -49,7 +49,7 @@ primary prompt, ``>>>``. (It shouldn't take long.) Numbers ------- -The interpreter acts as a simple calculator: you can type an expression at it +The interpreter acts as a simple calculator: you can type an expression into it and it will write the value. Expression syntax is straightforward: the operators ``+``, ``-``, ``*`` and ``/`` can be used to perform arithmetic; parentheses (``()``) can be used for grouping. @@ -184,11 +184,11 @@ If you don't want characters prefaced by ``\`` to be interpreted as special characters, you can use *raw strings* by adding an ``r`` before the first quote:: - >>> print('C:\some\name') # here \n means newline! - C:\some + >>> print('C:\this\name') # here \t means tab, \n means newline + C: his ame - >>> print(r'C:\some\name') # note the r before the quote - C:\some\name + >>> print(r'C:\this\name') # note the r before the quote + C:\this\name There is one subtle aspect to raw strings: a raw string may not end in an odd number of ``\`` characters; see diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index d83ecca270b..e7c64104474 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -1,13 +1,13 @@ .. _tut-brieftour: ********************************** -Brief Tour of the Standard Library +Brief tour of the standard library ********************************** .. _tut-os-interface: -Operating System Interface +Operating system interface ========================== The :mod:`os` module provides dozens of functions for interacting with the @@ -47,7 +47,7 @@ a higher level interface that is easier to use:: .. _tut-file-wildcards: -File Wildcards +File wildcards ============== The :mod:`glob` module provides a function for making file lists from directory @@ -60,7 +60,7 @@ wildcard searches:: .. _tut-command-line-arguments: -Command Line Arguments +Command-line arguments ====================== Common utility scripts often need to process command line arguments. These @@ -97,7 +97,7 @@ to ``['alpha.txt', 'beta.txt']``. .. _tut-stderr: -Error Output Redirection and Program Termination +Error output redirection and program termination ================================================ The :mod:`sys` module also has attributes for *stdin*, *stdout*, and *stderr*. @@ -112,7 +112,7 @@ The most direct way to terminate a script is to use ``sys.exit()``. .. _tut-string-pattern-matching: -String Pattern Matching +String pattern matching ======================= The :mod:`re` module provides regular expression tools for advanced string @@ -175,7 +175,7 @@ computations. .. _tut-internet-access: -Internet Access +Internet access =============== There are a number of modules for accessing the internet and processing internet @@ -183,13 +183,13 @@ protocols. Two of the simplest are :mod:`urllib.request` for retrieving data from URLs and :mod:`smtplib` for sending mail:: >>> from urllib.request import urlopen - >>> with urlopen('http://worldtimeapi.org/api/timezone/etc/UTC.txt') as response: + >>> with urlopen('https://docs.python.org/3/') as response: ... for line in response: ... line = line.decode() # Convert bytes to a str - ... if line.startswith('datetime'): + ... if 'updated' in line: ... print(line.rstrip()) # Remove trailing newline ... - datetime: 2022-01-01T01:36:47.689215+00:00 + Last updated on Nov 11, 2025 (20:11 UTC). >>> import smtplib >>> server = smtplib.SMTP('localhost') @@ -206,7 +206,7 @@ from URLs and :mod:`smtplib` for sending mail:: .. _tut-dates-and-times: -Dates and Times +Dates and times =============== The :mod:`datetime` module supplies classes for manipulating dates and times in @@ -216,15 +216,15 @@ formatting and manipulation. The module also supports objects that are timezone aware. :: >>> # dates are easily constructed and formatted - >>> from datetime import date - >>> now = date.today() + >>> import datetime as dt + >>> now = dt.date.today() >>> now datetime.date(2003, 12, 2) >>> now.strftime("%m-%d-%y. %d %b %Y is a %A on the %d day of %B.") '12-02-03. 02 Dec 2003 is a Tuesday on the 02 day of December.' >>> # dates support calendar arithmetic - >>> birthday = date(1964, 7, 31) + >>> birthday = dt.date(1964, 7, 31) >>> age = now - birthday >>> age.days 14368 @@ -232,7 +232,7 @@ aware. :: .. _tut-data-compression: -Data Compression +Data compression ================ Common data archiving and compression formats are directly supported by modules @@ -254,7 +254,7 @@ including: :mod:`zlib`, :mod:`gzip`, :mod:`bz2`, :mod:`lzma`, :mod:`zipfile` and .. _tut-performance-measurement: -Performance Measurement +Performance measurement ======================= Some Python users develop a deep interest in knowing the relative performance of @@ -278,7 +278,7 @@ larger blocks of code. .. _tut-quality-control: -Quality Control +Quality control =============== One approach for developing high quality software is to write tests for each @@ -324,7 +324,7 @@ file:: .. _tut-batteries-included: -Batteries Included +Batteries included ================== Python has a "batteries included" philosophy. This is best seen through the @@ -335,7 +335,7 @@ sophisticated and robust capabilities of its larger packages. For example: names, no direct knowledge or handling of XML is needed. * The :mod:`email` package is a library for managing email messages, including - MIME and other :rfc:`2822`-based message documents. Unlike :mod:`smtplib` and + MIME and other :rfc:`5322`-based message documents. Unlike :mod:`smtplib` and :mod:`poplib` which actually send and receive messages, the email package has a complete toolset for building or decoding complex message structures (including attachments) and for implementing internet encoding and header diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 678b71c9274..6c68ba01081 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -1,7 +1,7 @@ .. _tut-brieftourtwo: ********************************************** -Brief Tour of the Standard Library --- Part II +Brief tour of the standard library --- part II ********************************************** This second tour covers more advanced modules that support professional @@ -10,7 +10,7 @@ programming needs. These modules rarely occur in small scripts. .. _tut-output-formatting: -Output Formatting +Output formatting ================= The :mod:`reprlib` module provides a version of :func:`repr` customized for @@ -130,7 +130,7 @@ templates for XML files, plain text reports, and HTML web reports. .. _tut-binary-formats: -Working with Binary Data Record Layouts +Working with binary data record layouts ======================================= The :mod:`struct` module provides :func:`~struct.pack` and @@ -178,14 +178,13 @@ tasks in background while the main program continues to run:: class AsyncZip(threading.Thread): def __init__(self, infile, outfile): - threading.Thread.__init__(self) + super().__init__() self.infile = infile self.outfile = outfile def run(self): - f = zipfile.ZipFile(self.outfile, 'w', zipfile.ZIP_DEFLATED) - f.write(self.infile) - f.close() + with zipfile.ZipFile(self.outfile, 'w', zipfile.ZIP_DEFLATED) as f: + f.write(self.infile) print('Finished background zip of:', self.infile) background = AsyncZip('mydata.txt', 'myarchive.zip') @@ -245,7 +244,7 @@ application. .. _tut-weak-references: -Weak References +Weak references =============== Python does automatic memory management (reference counting for most objects and @@ -286,7 +285,7 @@ applications include caching objects that are expensive to create:: .. _tut-list-tools: -Tools for Working with Lists +Tools for working with lists ============================ Many data structure needs can be met with the built-in list type. However, @@ -352,7 +351,7 @@ not want to run a full list sort:: .. _tut-decimal-fp: -Decimal Floating-Point Arithmetic +Decimal floating-point arithmetic ================================= The :mod:`decimal` module offers a :class:`~decimal.Decimal` datatype for diff --git a/Doc/tutorial/whatnow.rst b/Doc/tutorial/whatnow.rst index dbe2d7fc099..aae8f29b007 100644 --- a/Doc/tutorial/whatnow.rst +++ b/Doc/tutorial/whatnow.rst @@ -30,7 +30,7 @@ the set are: More Python resources: -* https://www.python.org: The major Python web site. It contains code, +* https://www.python.org: The major Python website. It contains code, documentation, and pointers to Python-related pages around the web. * https://docs.python.org: Fast access to Python's documentation. @@ -68,6 +68,6 @@ already contain the solution for your problem. .. rubric:: Footnotes -.. [#] "Cheese Shop" is a Monty Python's sketch: a customer enters a cheese shop, +.. [#] "Cheese Shop" is a Monty Python sketch: a customer enters a cheese shop, but whatever cheese he asks for, the clerk says it's missing. diff --git a/Doc/using/android.rst b/Doc/using/android.rst index cb762310328..60a13569318 100644 --- a/Doc/using/android.rst +++ b/Doc/using/android.rst @@ -30,7 +30,7 @@ Adding Python to an Android app Most app developers should use one of the following tools, which will provide a much easier experience: -* `Briefcase `__, from the BeeWare project +* `Briefcase `__, from the BeeWare project * `Buildozer `__, from the Kivy project * `Chaquopy `__ * `pyqtdeploy `__ @@ -40,8 +40,15 @@ If you're sure you want to do all of this manually, read on. You can use the :source:`testbed app ` as a guide; each step below contains a link to the relevant file. -* Build Python by following the instructions in :source:`Android/README.md`. - This will create the directory ``cross-build/HOST/prefix``. +* First, acquire a build of Python for Android: + + * The easiest way is to download an Android release from `python.org + `__. The ``prefix`` directory + mentioned below is at the top level of the package. + + * Or if you want to build it yourself, follow the instructions in + :source:`Android/README.md`. The ``prefix`` directory will be created under + :samp:`cross-build/{HOST}`. * Add code to your :source:`build.gradle ` file to copy the following items into your project. All except your own Python diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 448f2725e9a..7cbc03f5f12 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -49,7 +49,7 @@ additional methods of invocation: appropriately named script from that directory. * When called with ``-c command``, it executes the Python statement(s) given as *command*. Here *command* may contain multiple statements separated by - newlines. Leading whitespace is significant in Python statements! + newlines. * When called with ``-m module-name``, the given module is located on the Python module path and executed as a script. @@ -254,6 +254,15 @@ Miscellaneous options .. versionchanged:: 3.5 Affects also comparisons of :class:`bytes` with :class:`int`. + .. deprecated:: 3.15 + + Deprecate :option:`-b` and :option:`!-bb` command line options + and schedule them to become no-op in Python 3.17. + These were primarily helpers for the Python 2 -> 3 transition. + Starting with Python 3.17, no :exc:`BytesWarning` will be raised + for these cases; use a type checker instead. + + .. option:: -B If given, Python won't try to write ``.pyc`` files on the @@ -381,7 +390,7 @@ Miscellaneous options Hash randomization is intended to provide protection against a denial-of-service caused by carefully chosen inputs that exploit the worst case performance of a dict construction, *O*\ (*n*\ :sup:`2`) complexity. See - http://ocert.org/advisories/ocert-2011-003.html for details. + https://ocert.org/advisories/ocert-2011-003.html for details. :envvar:`PYTHONHASHSEED` allows you to set a fixed value for the hash seed secret. @@ -470,8 +479,10 @@ Miscellaneous options The *action* field is as explained above but only applies to warnings that match the remaining fields. - The *message* field must match the whole warning message; this match is - case-insensitive. + The *message* field must match the start of the warning message; + this match is case-insensitive. + If it starts and ends with a forward slash (``/``), it specifies + a regular expression, otherwise it specifies a literal string. The *category* field matches the warning category (ex: ``DeprecationWarning``). This must be a class name; the match test @@ -480,6 +491,10 @@ Miscellaneous options The *module* field matches the (fully qualified) module name; this match is case-sensitive. + If it starts and ends with a forward slash (``/``), it specifies + a regular expression that the start of the fully qualified module name + must match, otherwise it specifies a literal string that the fully + qualified module name must be equal to. The *lineno* field matches the line number, where zero matches all line numbers and is thus equivalent to an omitted line number. @@ -497,6 +512,9 @@ Miscellaneous options See :ref:`warning-filter` and :ref:`describing-warning-filters` for more details. + .. versionchanged:: 3.15 + Added regular expression support for *message* and *module*. + .. option:: -x @@ -636,13 +654,17 @@ Miscellaneous options .. versionadded:: 3.13 - * :samp:`-X presite={package.module}` specifies a module that should be - imported before the :mod:`site` module is executed and before the + * :samp:`-X presite={module}` or :samp:`-X presite={module:func}` specifies + an entry point that should be executed before the :mod:`site` module is + executed and before the :mod:`__main__` module exists. Therefore, the imported module isn't :mod:`__main__`. This can be used to execute code early during Python initialization. Python needs to be :ref:`built in debug mode ` for this option to exist. See also :envvar:`PYTHON_PRESITE`. + .. versionchanged:: next + Accept also ``module:func`` entry point format. + .. versionadded:: 3.13 * :samp:`-X gil={0,1}` forces the GIL to be disabled or enabled, @@ -669,6 +691,13 @@ Miscellaneous options .. versionadded:: 3.14 + * :samp:`-X pathconfig_warnings={0,1}` if true (``1``) then + :ref:`sys-path-init` is allowed to log warnings into stderr. + If false (``0``) suppress these warnings. Set to true by default. + See also :envvar:`PYTHON_PATHCONFIG_WARNINGS`. + + .. versionadded:: 3.15 + * :samp:`-X tlbc={0,1}` enables (1, the default) or disables (0) thread-local bytecode in builds configured with :option:`--disable-gil`. When disabled, this also disables the specializing interpreter. See also @@ -676,6 +705,14 @@ Miscellaneous options .. versionadded:: 3.14 + * :samp:`-X lazy_imports={all,none,normal}` controls lazy import behavior. + ``all`` makes all imports lazy by default, ``none`` disables lazy imports + entirely (even explicit ``lazy`` statements become eager), and ``normal`` + (the default) respects the ``lazy`` keyword in source code. + See also :envvar:`PYTHON_LAZY_IMPORTS`. + + .. versionadded:: 3.15 + It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -923,8 +960,9 @@ conflict. .. envvar:: PYTHONNOUSERSITE - If this is set, Python won't add the :data:`user site-packages directory - ` to :data:`sys.path`. + This is equivalent to the :option:`-s` option. If this is set, Python won't + add the :data:`user site-packages directory ` to + :data:`sys.path`. .. seealso:: @@ -938,6 +976,9 @@ conflict. and :ref:`installation paths ` for ``python -m pip install --user``. + To disable the user site-packages, see :envvar:`PYTHONNOUSERSITE` or the :option:`-s` + option. + .. seealso:: :pep:`370` -- Per user site-packages directory @@ -971,6 +1012,9 @@ conflict. See :ref:`warning-filter` and :ref:`describing-warning-filters` for more details. + .. versionchanged:: 3.15 + Added regular expression support for *message* and *module*. + .. envvar:: PYTHONFAULTHANDLER @@ -1045,6 +1089,13 @@ conflict. * ``pymalloc_debug``: same as ``pymalloc`` but also install debug hooks. * ``mimalloc_debug``: same as ``mimalloc`` but also install debug hooks. + .. note:: + + In the :term:`free-threaded ` build, the ``malloc``, + ``malloc_debug``, ``pymalloc``, and ``pymalloc_debug`` values are not + supported. Only ``default``, ``debug``, ``mimalloc``, and + ``mimalloc_debug`` are accepted. + .. versionadded:: 3.6 .. versionchanged:: 3.7 @@ -1054,18 +1105,48 @@ conflict. .. envvar:: PYTHONMALLOCSTATS If set to a non-empty string, Python will print statistics of the - :ref:`pymalloc memory allocator ` every time a new pymalloc object - arena is created, and on shutdown. + :ref:`pymalloc memory allocator ` or the + :ref:`mimalloc memory allocator ` (whichever is in use) + every time a new object arena is created, and on shutdown. This variable is ignored if the :envvar:`PYTHONMALLOC` environment variable is used to force the :c:func:`malloc` allocator of the C library, or if - Python is configured without ``pymalloc`` support. + Python is configured without both ``pymalloc`` and ``mimalloc`` support. .. versionchanged:: 3.6 This variable can now also be used on Python compiled in release mode. It now has no effect if set to an empty string. +.. envvar:: PYTHON_PYMALLOC_HUGEPAGES + + If set to a non-zero integer, enable huge page support for + :ref:`pymalloc ` arenas. Set to ``0`` or unset to disable. + Python must be compiled with :option:`--with-pymalloc-hugepages` for this + variable to have any effect. + + When enabled, arena allocation uses ``MAP_HUGETLB`` (Linux) or + ``MEM_LARGE_PAGES`` (Windows) with automatic fallback to regular pages if + huge pages are not available. + + .. warning:: + + On Linux, if the huge-page pool is exhausted, page faults — including + copy-on-write faults triggered by :func:`os.fork` — deliver ``SIGBUS`` + and kill the process. Only enable this in environments where the + huge-page pool is properly sized and fork-safety is not a concern. + + On Windows you need a special privilege. See the + `Windows documentation for large pages + `_ + for details. Python will fail on startup if the required privilege + `SeLockMemoryPrivilege + `_ + is not held by the user. + + .. versionadded:: 3.15 + + .. envvar:: PYTHONLEGACYWINDOWSFSENCODING If set to a non-empty string, the default :term:`filesystem encoding and @@ -1256,12 +1337,18 @@ conflict. .. envvar:: PYTHON_BASIC_REPL If this variable is set to any value, the interpreter will not attempt to - load the Python-based :term:`REPL` that requires :mod:`curses` and - :mod:`readline`, and will instead use the traditional parser-based - :term:`REPL`. + load the Python-based :term:`REPL` that requires :mod:`readline`, and will + instead use the traditional parser-based :term:`REPL`. .. versionadded:: 3.13 +.. envvar:: PYTHON_BASIC_COMPLETER + + If this variable is set to any value, PyREPL will use :mod:`rlcompleter` to + implement tab completion, instead of the default one which uses colors. + + .. versionadded:: 3.15 + .. envvar:: PYTHON_HISTORY This environment variable can be used to set the location of a @@ -1301,6 +1388,14 @@ conflict. .. versionadded:: 3.14 +.. envvar:: PYTHON_PATHCONFIG_WARNINGS + + If true (``1``) then :ref:`sys-path-init` is allowed to log warnings into + stderr. If false (``0``) suppress these warnings. Set to true by default. + See also :option:`-X pathconfig_warnings<-X>`. + + .. versionadded:: 3.15 + .. envvar:: PYTHON_JIT On builds where experimental just-in-time compilation is available, this @@ -1319,6 +1414,17 @@ conflict. .. versionadded:: 3.14 +.. envvar:: PYTHON_LAZY_IMPORTS + + Controls lazy import behavior. Accepts three values: ``all`` makes all + imports lazy by default, ``none`` disables lazy imports entirely (even + explicit ``lazy`` statements become eager), and ``normal`` (the default) + respects the ``lazy`` keyword in source code. + + See also the :option:`-X lazy_imports <-X>` command-line option. + + .. versionadded:: 3.15 + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ @@ -1356,4 +1462,7 @@ Debug-mode variables Needs Python configured with the :option:`--with-pydebug` build option. + .. versionchanged:: next + Accept also ``module:func`` entry point format. + .. versionadded:: 3.13 diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 6eb5c243b87..086f6bfa22a 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -4,10 +4,13 @@ Configure Python .. highlight:: sh + +.. _build-requirements: + Build Requirements ================== -Features and minimum versions required to build CPython: +To build CPython, you will need: * A `C11 `_ compiler. `Optional C11 features @@ -22,52 +25,138 @@ Features and minimum versions required to build CPython: * Support for threads. -* OpenSSL 1.1.1 is the minimum version and OpenSSL 3.0.16 is the recommended - minimum version for the :mod:`ssl` and :mod:`hashlib` extension modules. - -* SQLite 3.15.2 for the :mod:`sqlite3` extension module. - -* Tcl/Tk 8.5.12 for the :mod:`tkinter` module. - -* `libmpdec `_ 2.5.0 - for the :mod:`decimal` module. - -* Autoconf 2.72 and aclocal 1.16.5 are required to regenerate the - :file:`configure` script. - -.. versionchanged:: 3.1 - Tcl/Tk version 8.3.1 is now required. - .. versionchanged:: 3.5 On Windows, Visual Studio 2015 or later is now required. - Tcl/Tk version 8.4 is now required. .. versionchanged:: 3.6 - Selected C99 features are now required, like ```` and ``static - inline`` functions. + Selected C99 features, like ```` and ``static inline`` functions, + are now required. .. versionchanged:: 3.7 - Thread support and OpenSSL 1.0.2 are now required. - -.. versionchanged:: 3.10 - OpenSSL 1.1.1 is now required. - Require SQLite 3.7.15. + Thread support is now required. .. versionchanged:: 3.11 C11 compiler, IEEE 754 and NaN support are now required. On Windows, Visual Studio 2017 or later is required. - Tcl/Tk version 8.5.12 is now required for the :mod:`tkinter` module. - -.. versionchanged:: 3.13 - Autoconf 2.71, aclocal 1.16.5 and SQLite 3.15.2 are now required. - -.. versionchanged:: 3.14 - Autoconf 2.72 is now required. See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform support". +.. _optional-module-requirements: + +Requirements for optional modules +--------------------------------- + +Some :term:`optional modules ` of the standard library +require third-party libraries installed for development +(for example, header files must be available). + +Missing requirements are reported in the ``configure`` output. +Modules that are missing due to missing dependencies are listed near the end +of the ``make`` output, +sometimes using an internal name, for example, ``_ctypes`` for :mod:`ctypes` +module. + +If you distribute a CPython interpreter without optional modules, +it's best practice to advise users, who generally expect that +standard library modules are available. + +Dependencies to build optional modules are: + +.. list-table:: + :header-rows: 1 + :align: left + + * - Dependency + - Minimum version + - Python module + * - `libbz2 `_ + - + - :mod:`bz2` + * - `libffi `_ + - 3.3.0 recommended + - :mod:`ctypes` + * - `liblzma `_ + - + - :mod:`lzma` + * - `libmpdec `_ + - 2.5.0 + - :mod:`decimal` [1]_ + * - `libreadline `_ or + `libedit `_ [2]_ + - + - :mod:`readline` + * - `libuuid `_ + - + - ``_uuid`` [3]_ + * - `ncurses `_ [4]_ + - + - :mod:`curses` + * - `OpenSSL `_ + - | 3.0.18 recommended + | (1.1.1 minimum) + - :mod:`ssl`, :mod:`hashlib` [5]_ + * - `SQLite `_ + - 3.15.2 + - :mod:`sqlite3` + * - `Tcl/Tk `_ + - 8.5.12 + - :mod:`tkinter`, :ref:`IDLE `, :mod:`turtle` + * - `zlib `_ + - 1.2.2.1 + - :mod:`zlib`, :mod:`gzip`, :mod:`ensurepip` + * - `zstd `_ + - 1.4.5 + - :mod:`compression.zstd` + +.. [1] If *libmpdec* is not available, the :mod:`decimal` module will use + a pure-Python implementation. + See :option:`--with-system-libmpdec` for details. +.. [2] See :option:`--with-readline` for choosing the backend for the + :mod:`readline` module. +.. [3] The :mod:`uuid` module uses ``_uuid`` to generate "safe" UUIDs. + See the module documentation for details. +.. [4] The :mod:`curses` module requires the ``libncurses`` or ``libncursesw`` + library. + The :mod:`curses.panel` module additionally requires the ``libpanel`` or + ``libpanelw`` library. +.. [5] If OpenSSL is not available, the :mod:`hashlib` module will use + bundled implementations of several hash functions. + See :option:`--with-builtin-hashlib-hashes` for *forcing* usage of OpenSSL. + +Note that the table does not include all optional modules; in particular, +platform-specific modules like :mod:`winreg` are not listed here. + +.. seealso:: + + * The `devguide `_ + includes a full list of dependencies required to build all modules and + instructions on how to install them on common platforms. + * :option:`--with-system-expat` allows building with an external + `libexpat `_ library. + * :ref:`configure-options-for-dependencies` + +.. versionchanged:: 3.1 + Tcl/Tk version 8.3.1 is now required for :mod:`tkinter`. + +.. versionchanged:: 3.5 + Tcl/Tk version 8.4 is now required for :mod:`tkinter`. + +.. versionchanged:: 3.7 + OpenSSL 1.0.2 is now required for :mod:`hashlib` and :mod:`ssl`. + +.. versionchanged:: 3.10 + OpenSSL 1.1.1 is now required for :mod:`hashlib` and :mod:`ssl`. + SQLite 3.7.15 is now required for :mod:`sqlite3`. + +.. versionchanged:: 3.11 + Tcl/Tk version 8.5.12 is now required for :mod:`tkinter`. + +.. versionchanged:: 3.13 + SQLite 3.15.2 is now required for :mod:`sqlite3`. + + Generated files =============== @@ -94,8 +183,19 @@ The container is optional, the following command can be run locally:: autoreconf -ivf -Werror -The generated files can change depending on the exact ``autoconf-archive``, -``aclocal`` and ``pkg-config`` versions. +The generated files can change depending on the exact versions of the +tools used. +The container that CPython uses has +`Autoconf `_ 2.72, +``aclocal`` from `Automake `_ 1.16.5, +and `pkg-config `_ 1.8.1. + +.. versionchanged:: 3.13 + Autoconf 2.71 and aclocal 1.16.5 and are now used to regenerate + :file:`configure`. + +.. versionchanged:: 3.14 + Autoconf 2.72 is now used to regenerate :file:`configure`. .. _configure-options: @@ -222,6 +322,30 @@ General Options .. versionadded:: 3.11 +.. option:: --with-missing-stdlib-config=FILE + + Path to a `JSON `_ configuration file + containing custom error messages for missing :term:`standard library` modules. + + This option is intended for Python distributors who wish to provide + distribution-specific guidance when users encounter standard library + modules that are missing or packaged separately. + + The JSON file should map missing module names to custom error message strings. + For example, if your distribution packages :mod:`tkinter` and + :mod:`_tkinter` separately and excludes :mod:`!_gdbm` for legal reasons, + the configuration could contain: + + .. code-block:: json + + { + "_gdbm": "The '_gdbm' module is not available in this distribution", + "tkinter": "Install the python-tk package to use tkinter", + "_tkinter": "Install the python-tk package to use tkinter", + } + + .. versionadded:: 3.15 + .. option:: --enable-pystats Turn on internal Python performance statistics gathering. @@ -297,7 +421,7 @@ General Options :no-typesetting: Enables support for running Python without the :term:`global interpreter - lock` (GIL): free threading build. + lock` (GIL): :term:`free-threaded build`. Defines the ``Py_GIL_DISABLED`` macro and adds ``"t"`` to :data:`sys.abiflags`. @@ -339,6 +463,17 @@ General Options ``pkg-config`` options. +.. option:: --disable-epoll + + Build without ``epoll``, meaning that :py:func:`select.epoll` will not be + present even if the system provides an + :manpage:`epoll_create ` function. + This may be used on systems where :manpage:`!epoll_create` or + :manpage:`epoll_create1 ` is available + but incompatible with Linux semantics. + + .. versionadded:: 3.15 + C compiler options ------------------ @@ -376,6 +511,8 @@ Linker options Name for machine-dependent library files. +.. _configure-options-for-dependencies: + Options for third-party dependencies ------------------------------------ @@ -398,12 +535,6 @@ Options for third-party dependencies C compiler and linker flags for ``gdbm``. -.. option:: LIBB2_CFLAGS -.. option:: LIBB2_LIBS - - C compiler and linker flags for ``libb2`` (:ref:`BLAKE2 `), - used by :mod:`hashlib` module, overriding ``pkg-config``. - .. option:: LIBEDIT_CFLAGS .. option:: LIBEDIT_LIBS @@ -649,11 +780,32 @@ also be used to improve performance. .. versionadded:: 3.14 +.. option:: --without-frame-pointers + + Disable frame pointers, which are enabled by default (see :pep:`831`). + + By default, the build appends ``-fno-omit-frame-pointer`` (and + ``-mno-omit-leaf-frame-pointer`` when the compiler supports it) to + ``BASECFLAGS`` so profilers, debuggers, and system tracing tools + (``perf``, ``eBPF``, ``dtrace``, ``gdb``) can walk the C call stack + without DWARF metadata. The flags propagate to third-party C + extensions through :mod:`sysconfig`. On compilers that do not + understand them, the build silently skips them. + + Downstream packagers and authors of native libraries built with + custom build systems should set the same flags so the unwind chain + stays unbroken across all native frames. + + .. versionadded:: 3.15 + .. option:: --without-mimalloc Disable the fast :ref:`mimalloc ` allocator (enabled by default). + This option cannot be used together with :option:`--disable-gil` + because the :term:`free-threaded ` build requires mimalloc. + See also :envvar:`PYTHONMALLOC` environment variable. .. option:: --without-pymalloc @@ -663,6 +815,36 @@ also be used to improve performance. See also :envvar:`PYTHONMALLOC` environment variable. +.. option:: --with-pymalloc-hugepages + + Enable huge page support for :ref:`pymalloc ` arenas (disabled by + default). When enabled, the arena size on 64-bit platforms is increased to + 2 MiB and arena allocation uses ``MAP_HUGETLB`` (Linux) or + ``MEM_LARGE_PAGES`` (Windows) with automatic fallback to regular pages. + + Even when compiled with this option, huge pages are **not** used at runtime + unless the :envvar:`PYTHON_PYMALLOC_HUGEPAGES` environment variable is set + to ``1``. This opt-in is required because huge pages + + * carry risks on Linux: if the huge-page pool is exhausted, page faults + (including copy-on-write faults after :func:`os.fork`) deliver ``SIGBUS`` + and kill the process. + + * need a special privilege on Windows. See the `Windows documentation for large pages + `_ + for details. Python will fail on startup if the required privilege + `SeLockMemoryPrivilege + `_ + is not held by the user. + + The configure script checks that the platform supports ``MAP_HUGETLB`` + and emits a warning if it is not available. + + On Windows, use the ``--pymalloc-hugepages`` flag with ``build.bat`` or + set the ``UsePymallocHugepages`` MSBuild property. + + .. versionadded:: 3.15 + .. option:: --without-doc-strings Disable static documentation strings to reduce the memory footprint (enabled @@ -742,9 +924,11 @@ See also the :ref:`Python Development Mode ` and the :option:`--with-trace-refs` configure option. .. versionchanged:: 3.8 - Release builds and debug builds are now ABI compatible: defining the + Release builds are now ABI compatible with debug builds: defining the ``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro (see the - :option:`--with-trace-refs` option). + :option:`--with-trace-refs` option). However, debug builds still expose + more symbols than release builds and code built against a debug build is not + necessarily compatible with a release build. Debug options @@ -846,6 +1030,21 @@ Linker options .. versionadded:: 3.10 +.. option:: --enable-static-libpython-for-interpreter + + Do not link the Python interpreter binary (``python3``) against the + shared Python library; instead, statically link the interpreter + against ``libpython`` as if ``--enable-shared`` had not been used, + but continue to build the shared ``libpython`` (for use by other + programs). + + This option does nothing if ``--enable-shared`` is not used. + + The default (when ``-enable-shared`` is used) is to link the Python + interpreter against the built shared library. + + .. versionadded:: next + Libraries options ----------------- @@ -869,9 +1068,16 @@ Libraries options .. versionchanged:: 3.13 Default to using the installed ``mpdecimal`` library. - .. deprecated-removed:: 3.13 3.15 + .. versionchanged:: 3.15 + + A bundled copy of the library will no longer be selected + implicitly if an installed ``mpdecimal`` library is not found. + In Python 3.15 only, it can still be selected explicitly using + ``--with-system-libmpdec=no`` or ``--without-system-libmpdec``. + + .. deprecated-removed:: 3.13 3.16 A copy of the ``mpdecimal`` library sources will no longer be distributed - with Python 3.15. + with Python 3.16. .. seealso:: :option:`LIBMPDEC_CFLAGS` and :option:`LIBMPDEC_LIBS`. @@ -1401,6 +1607,12 @@ Compiler flags .. versionadded:: 3.7 +.. envvar:: CFLAGS_CEVAL + + Flags used to compile ``Python/ceval.c``. + + .. versionadded:: 3.14.5 + .. envvar:: CCSHARED Compiler flags used to build a shared library. @@ -1502,6 +1714,9 @@ Linker flags value to be able to build extension modules using the directories specified in the environment variables. + Please consider using ``EXE_LDFLAGS`` if the supplied linker flags are + executable specific, e.g. GCC's ``-pie`` flag. + .. envvar:: LIBS Linker flags to pass libraries to the linker when linking the Python @@ -1537,6 +1752,30 @@ Linker flags .. versionadded:: 3.8 +.. envvar:: EXE_LDFLAGS + + Linker flags used for building executable targets such as the + interpreter. If supplied, :envvar:`PY_CORE_EXE_LDFLAGS` + will be used in replacement of :envvar:`PY_CORE_LDFLAGS`. + + .. versionadded:: 3.15 + +.. envvar:: CONFIGURE_EXE_LDFLAGS + + Value of :envvar:`EXE_LDFLAGS` variable passed to the ``./configure`` + script. + + .. versionadded:: 3.15 + +.. envvar:: PY_CORE_EXE_LDFLAGS + + Linker flags used for building the interpreter and + executable targets. + + Default: ``$(PY_CORE_LDFLAGS)`` + + .. versionadded:: 3.15 + .. rubric:: Footnotes diff --git a/Doc/using/ios.rst b/Doc/using/ios.rst index 91cfed16f0e..5e4033fb6ce 100644 --- a/Doc/using/ios.rst +++ b/Doc/using/ios.rst @@ -170,7 +170,7 @@ helpful. To add Python to an iOS Xcode project: 1. Build or obtain a Python ``XCFramework``. See the instructions in - :source:`iOS/README.rst` (in the CPython source distribution) for details on + :source:`Apple/iOS/README.md` (in the CPython source distribution) for details on how to build a Python ``XCFramework``. At a minimum, you will need a build that supports ``arm64-apple-ios``, plus one of either ``arm64-apple-ios-simulator`` or ``x86_64-apple-ios-simulator``. @@ -180,22 +180,19 @@ To add Python to an iOS Xcode project: of your project; however, you can use any other location that you want by adjusting paths as needed. -3. Drag the ``iOS/Resources/dylib-Info-template.plist`` file into your project, - and ensure it is associated with the app target. - -4. Add your application code as a folder in your Xcode project. In the +3. Add your application code as a folder in your Xcode project. In the following instructions, we'll assume that your user code is in a folder named ``app`` in the root of your project; you can use any other location by adjusting paths as needed. Ensure that this folder is associated with your app target. -5. Select the app target by selecting the root node of your Xcode project, then +4. Select the app target by selecting the root node of your Xcode project, then the target name in the sidebar that appears. -6. In the "General" settings, under "Frameworks, Libraries and Embedded +5. In the "General" settings, under "Frameworks, Libraries and Embedded Content", add ``Python.xcframework``, with "Embed & Sign" selected. -7. In the "Build Settings" tab, modify the following: +6. In the "Build Settings" tab, modify the following: - Build Options @@ -211,86 +208,24 @@ To add Python to an iOS Xcode project: * Quoted Include In Framework Header: No -8. Add a build step that copies the Python standard library into your app. In - the "Build Phases" tab, add a new "Run Script" build step *before* the - "Embed Frameworks" step, but *after* the "Copy Bundle Resources" step. Name - the step "Install Target Specific Python Standard Library", disable the - "Based on dependency analysis" checkbox, and set the script content to: +7. Add a build step that processes the Python standard library, and your own + Python binary dependencies. In the "Build Phases" tab, add a new "Run + Script" build step *before* the "Embed Frameworks" step, but *after* the + "Copy Bundle Resources" step. Name the step "Process Python libraries", + disable the "Based on dependency analysis" checkbox, and set the script + content to: .. code-block:: bash - set -e + set -e + source $PROJECT_DIR/Python.xcframework/build/build_utils.sh + install_python Python.xcframework app - mkdir -p "$CODESIGNING_FOLDER_PATH/python/lib" - if [ "$EFFECTIVE_PLATFORM_NAME" = "-iphonesimulator" ]; then - echo "Installing Python modules for iOS Simulator" - rsync -au --delete "$PROJECT_DIR/Python.xcframework/ios-arm64_x86_64-simulator/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" - else - echo "Installing Python modules for iOS Device" - rsync -au --delete "$PROJECT_DIR/Python.xcframework/ios-arm64/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" - fi + If you have placed your XCframework somewhere other than the root of your + project, modify the path to the first argument. - Note that the name of the simulator "slice" in the XCframework may be - different, depending the CPU architectures your ``XCFramework`` supports. - -9. Add a second build step that processes the binary extension modules in the - standard library into "Framework" format. Add a "Run Script" build step - *directly after* the one you added in step 8, named "Prepare Python Binary - Modules". It should also have "Based on dependency analysis" unchecked, with - the following script content: - - .. code-block:: bash - - set -e - - install_dylib () { - INSTALL_BASE=$1 - FULL_EXT=$2 - - # The name of the extension file - EXT=$(basename "$FULL_EXT") - # The location of the extension file, relative to the bundle - RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} - # The path to the extension file, relative to the install base - PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/} - # The full dotted name of the extension module, constructed from the file path. - FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d "." -f 1 | tr "/" "."); - # A bundle identifier; not actually used, but required by Xcode framework packaging - FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr "_" "-") - # The name of the framework folder. - FRAMEWORK_FOLDER="Frameworks/$FULL_MODULE_NAME.framework" - - # If the framework folder doesn't exist, create it. - if [ ! -d "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" ]; then - echo "Creating framework for $RELATIVE_EXT" - mkdir -p "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" - cp "$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" - plutil -replace CFBundleExecutable -string "$FULL_MODULE_NAME" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" - plutil -replace CFBundleIdentifier -string "$FRAMEWORK_BUNDLE_ID" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" - fi - - echo "Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME" - mv "$FULL_EXT" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" - # Create a placeholder .fwork file where the .so was - echo "$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" > ${FULL_EXT%.so}.fwork - # Create a back reference to the .so file location in the framework - echo "${RELATIVE_EXT%.so}.fwork" > "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin" - } - - PYTHON_VER=$(ls -1 "$CODESIGNING_FOLDER_PATH/python/lib") - echo "Install Python $PYTHON_VER standard library extension modules..." - find "$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload" -name "*.so" | while read FULL_EXT; do - install_dylib python/lib/$PYTHON_VER/lib-dynload/ "$FULL_EXT" - done - - # Clean up dylib template - rm -f "$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist" - - echo "Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)..." - find "$CODESIGNING_FOLDER_PATH/Frameworks" -name "*.framework" -exec /usr/bin/codesign --force --sign "$EXPANDED_CODE_SIGN_IDENTITY" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der "{}" \; - -10. Add Objective C code to initialize and use a Python interpreter in embedded - mode. You should ensure that: +8. Add Objective C code to initialize and use a Python interpreter in embedded + mode. You should ensure that: * UTF-8 mode (:c:member:`PyPreConfig.utf8_mode`) is *enabled*; * Buffered stdio (:c:member:`PyConfig.buffered_stdio`) is *disabled*; @@ -309,22 +244,19 @@ To add Python to an iOS Xcode project: Your app's bundle location can be determined using ``[[NSBundle mainBundle] resourcePath]``. -Steps 8, 9 and 10 of these instructions assume that you have a single folder of +Steps 7 and 8 of these instructions assume that you have a single folder of pure Python application code, named ``app``. If you have third-party binary modules in your app, some additional steps will be required: * You need to ensure that any folders containing third-party binaries are - either associated with the app target, or copied in as part of step 8. Step 8 - should also purge any binaries that are not appropriate for the platform a - specific build is targeting (i.e., delete any device binaries if you're - building an app targeting the simulator). + either associated with the app target, or are explicitly copied as part of + step 7. Step 7 should also purge any binaries that are not appropriate for + the platform a specific build is targeting (i.e., delete any device binaries + if you're building an app targeting the simulator). -* Any folders that contain third-party binaries must be processed into - framework form by step 9. The invocation of ``install_dylib`` that processes - the ``lib-dynload`` folder can be copied and adapted for this purpose. - -* If you're using a separate folder for third-party packages, ensure that folder - is included as part of the :envvar:`PYTHONPATH` configuration in step 10. +* If you're using a separate folder for third-party packages, ensure that + folder is added to the end of the call to ``install_python`` in step 7, and + as part of the :envvar:`PYTHONPATH` configuration in step 8. * If any of the folders that contain third-party packages will contain ``.pth`` files, you should add that folder as a *site directory* (using @@ -334,25 +266,30 @@ modules in your app, some additional steps will be required: Testing a Python package ------------------------ -The CPython source tree contains :source:`a testbed project ` that +The CPython source tree contains :source:`a testbed project ` that is used to run the CPython test suite on the iOS simulator. This testbed can also be used as a testbed project for running your Python library's test suite on iOS. -After building or obtaining an iOS XCFramework (See :source:`iOS/README.rst` -for details), create a clone of the Python iOS testbed project by running: +After building or obtaining an iOS XCFramework (see :source:`Apple/iOS/README.md` +for details), create a clone of the Python iOS testbed project. If you used the +``Apple`` build script to build the XCframework, you can run: .. code-block:: bash - $ python iOS/testbed clone --framework --app --app app-testbed + $ python cross-build/iOS/testbed clone --app --app app-testbed -You will need to modify the ``iOS/testbed`` reference to point to that -directory in the CPython source tree; any folders specified with the ``--app`` -flag will be copied into the cloned testbed project. The resulting testbed will -be created in the ``app-testbed`` folder. In this example, the ``module1`` and -``module2`` would be importable modules at runtime. If your project has -additional dependencies, they can be installed into the -``app-testbed/iOSTestbed/app_packages`` folder (using ``pip install --target -app-testbed/iOSTestbed/app_packages`` or similar). +Or, if you've sourced your own XCframework, by running: + +.. code-block:: bash + + $ python Apple/testbed clone --platform iOS --framework --app --app app-testbed + +Any folders specified with the ``--app`` flag will be copied into the cloned +testbed project. The resulting testbed will be created in the ``app-testbed`` +folder. In this example, the ``module1`` and ``module2`` would be importable +modules at runtime. If your project has additional dependencies, they can be +installed into the ``app-testbed/Testbed/app_packages`` folder (using ``pip +install --target app-testbed/Testbed/app_packages`` or similar). You can then use the ``app-testbed`` folder to run the test suite for your app, For example, if ``module1.tests`` was the entry point to your test suite, you @@ -381,7 +318,7 @@ tab. Modify the "Arguments Passed On Launch" value to change the testing arguments. The test plan also disables parallel testing, and specifies the use of the -``iOSTestbed.lldbinit`` file for providing configuration of the debugger. The +``Testbed.lldbinit`` file for providing configuration of the debugger. The default debugger configuration disables automatic breakpoints on the ``SIGINT``, ``SIGUSR1``, ``SIGUSR2``, and ``SIGXFSZ`` signals. @@ -391,7 +328,12 @@ App Store Compliance The only mechanism for distributing apps to third-party iOS devices is to submit the app to the iOS App Store; apps submitted for distribution must pass Apple's app review process. This process includes a set of automated validation -rules that inspect the submitted application bundle for problematic code. +rules that inspect the submitted application bundle for problematic code. There +are some steps that must be taken to ensure that your app will be able to pass +these validation steps. + +Incompatible code in the standard library +----------------------------------------- The Python standard library contains some code that is known to violate these automated rules. While these violations appear to be false positives, Apple's @@ -402,3 +344,18 @@ The Python source tree contains :source:`a patch file ` that will remove all code that is known to cause issues with the App Store review process. This patch is applied automatically when building for iOS. + +Privacy manifests +----------------- + +In April 2025, Apple introduced a requirement for `certain third-party +libraries to provide a Privacy Manifest +`__. +As a result, if you have a binary module that uses one of the affected +libraries, you must provide an ``.xcprivacy`` file for that library. +OpenSSL is one library affected by this requirement, but there are others. + +If you produce a binary module named ``mymodule.so``, and use you the Xcode +build script described in step 7 above, you can place a ``mymodule.xcprivacy`` +file next to ``mymodule.so``, and the privacy manifest will be installed into +the required location when the binary module is converted into a framework. diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 2fd6eace2b5..6cf945de5b3 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -5,9 +5,6 @@ Using Python on macOS ********************* -.. sectionauthor:: Bob Savage -.. sectionauthor:: Ned Deily - This document aims to give an overview of macOS-specific behavior you should know about to get started with Python on Mac computers. Python on a Mac running macOS is very similar to Python on other Unix-derived platforms, diff --git a/Doc/using/mac_installer_07_applications.png b/Doc/using/mac_installer_07_applications.png index 940219cad6f..c8b3aa1099a 100644 Binary files a/Doc/using/mac_installer_07_applications.png and b/Doc/using/mac_installer_07_applications.png differ diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index 9ec4e341932..829bdfe8b11 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -6,9 +6,6 @@ Using Python on Unix platforms ******************************** -.. sectionauthor:: Shriphani Palakodety - - Getting and installing the latest version of Python =================================================== @@ -84,11 +81,17 @@ On FreeBSD and OpenBSD Building Python =============== +.. seealso:: + + If you want to contribute to CPython, refer to the + `devguide `_, + which includes build instructions and other tips on setting up environment. + If you want to compile CPython yourself, first thing you should do is get the `source `_. You can download either the -latest release's source or just grab a fresh `clone -`_. (If you want -to contribute patches, you will need a clone.) +latest release's source or grab a fresh `clone +`_. +You will also need to install the :ref:`build requirements `. The build process consists of the usual commands:: diff --git a/Doc/using/win_install_freethreaded.png b/Doc/using/win_install_freethreaded.png index 0aa01c1df6e..12b89c0165d 100644 Binary files a/Doc/using/win_install_freethreaded.png and b/Doc/using/win_install_freethreaded.png differ diff --git a/Doc/using/win_installer.png b/Doc/using/win_installer.png index 03bf2d7b16c..fc9605a79cf 100644 Binary files a/Doc/using/win_installer.png and b/Doc/using/win_installer.png differ diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 2cfdeac10f6..eea1e2f64a4 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -4,14 +4,14 @@ .. _Microsoft Store app: https://apps.microsoft.com/detail/9NQ7512CXL7T +.. _legacy launcher: https://www.python.org/ftp/python/3.14.0/win32/launcher.msi + .. _using-on-windows: ************************* Using Python on Windows ************************* -.. sectionauthor:: Steve Dower - This document aims to give an overview of Windows-specific behaviour you should know about when using Python on Microsoft Windows. @@ -60,7 +60,7 @@ packages. .. _windows-path-mod: .. _launcher: -Python Install Manager +Python install manager ====================== Installation @@ -103,7 +103,7 @@ Windows Server 2019, please see :ref:`pymanager-advancedinstall` below for more information. -Basic Use +Basic use --------- The recommended command for launching Python is ``python``, which will either @@ -124,9 +124,8 @@ is also an unambiguous ``pymanager`` command. Scripted installs that are intending to use Python install manager should consider using ``pymanager``, due to the lower chance of encountering a conflict with existing installs. The only difference between the two commands is when running without any arguments: -``py`` will install and launch your default interpreter, while ``pymanager`` -will display help (``pymanager exec ...`` provides equivalent behaviour to -``py ...``). +``py`` will launch your default interpreter, while ``pymanager`` will display +help (``pymanager exec ...`` provides equivalent behaviour to ``py ...``). Each of these commands also has a windowed version that avoids creating a console window. These are ``pyw``, ``pythonw`` and ``pymanagerw``. A ``python3`` @@ -185,15 +184,14 @@ that virtual environment. In this scenario, the ``python`` command was likely already overridden and none of these checks occurred. However, this behaviour ensures that the ``py`` command can be used interchangeably. -When you launch either ``python`` or ``py`` but do not have any runtimes -installed, and the requested version is the default, it will be installed -automatically and then launched. Otherwise, the requested version will be -installed if automatic installation is configured (most likely by setting -``PYTHON_MANAGER_AUTOMATIC_INSTALL`` to ``true``), or if the ``py exec`` or -``pymanager exec`` forms of the command were used. +When no runtimes are installed, any launch command will try to install the +requested version and launch it. However, after any version is installed, only +the ``py exec ...`` and ``pymanager exec ...`` commands will install if the +requested version is absent. Other forms of commands will display an error and +direct you to use ``py install`` first. -Command Help +Command help ------------ The ``py help`` command will display the full list of supported commands, along @@ -218,7 +216,7 @@ override multiple settings at once. See :ref:`pymanager-config` below for more information about these files. -Listing Runtimes +Listing runtimes ---------------- .. code:: @@ -259,7 +257,7 @@ For compatibility with the old launcher, the ``--list``, ``--list-paths``, additional options, and will produce legacy formatted output. -Installing Runtimes +Installing runtimes ------------------- .. code:: @@ -287,18 +285,36 @@ work. Passing ``--dry-run`` will generate output and logs, but will not modify any installs. +Passing ``--refresh`` will update all registrations for installed runtimes. This +will recreate Start menu shortcuts, registry keys, and global aliases (such as +``python3.14.exe`` or for any installed scripts). These are automatically +refreshed on installation of any runtime, but may need to be manually refreshed +after installing packages. + In addition to the above options, the ``--target`` option will extract the -runtime to the specified directory instead of doing a normal install. This is -useful for embedding runtimes into larger applications. +runtime to the specified directory instead of doing a normal install. +This is useful for embedding runtimes into larger applications. +Unlike a normal install, ``py`` will not be aware of the extracted runtime, +and no Start menu or other shortcuts will be created. +To launch the runtime, directly execute the main executable (typically +``python.exe``) in the target directory. .. code:: $> py install ... [-t=|--target=] +The ``py exec`` command will install the requested runtime if it is not already +present. This is controlled by the ``automatic_install`` configuration +(:envvar:`PYTHON_MANAGER_AUTOMATIC_INSTALL`), and is enabled by default. +If no runtimes are available at all, all launch commands will do an automatic +install if the configuration setting allows. This is to ensure a good experience +for new users, but should not generally be relied on rather than using the +``py exec`` command or explicit install commands. + .. _pymanager-offline: -Offline Installs +Offline installs ---------------- To perform offline installs of Python, you will need to first create an offline @@ -330,7 +346,7 @@ In this way, Python runtimes can be installed and managed on a machine without access to the internet. -Uninstalling Runtimes +Uninstalling runtimes --------------------- .. code:: @@ -376,10 +392,13 @@ overridden installs may resolve settings differently. A global configuration file may be configured by an administrator, and would be read first. The user configuration file is stored at -:file:`%AppData%\\Python\\pymanager.json` (by default) and is read next, +:file:`%AppData%\\Python\\pymanager.json` +(note that this location is under ``Roaming``, not ``Local``) and is read next, overwriting any settings from earlier files. An additional configuration file may be specified as the ``PYTHON_MANAGER_CONFIG`` environment variable or the ``--config`` command line option (but not both). +These locations may be modified by administrative customization options listed +later. The following settings are those that are considered likely to be modified in normal use. Later sections list those that are intended for administrative @@ -417,9 +436,11 @@ customization. By default, :file:`%TEMP%`. * - ``automatic_install`` - - ``PYTHON_MANAGER_AUTOMATIC_INSTALL`` - - True to allow automatic installs when specifying a particular runtime - to launch. + - .. envvar:: PYTHON_MANAGER_AUTOMATIC_INSTALL + - True to allow automatic installs when using ``py exec`` to launch (or + ``py`` when no runtimes are installed yet). + Other commands will not automatically install, regardless of this + setting. By default, true. * - ``include_unmanaged`` @@ -452,11 +473,39 @@ customization. - ``PYTHON_MANAGER_SOURCE_URL`` - Override the index feed to obtain new installs from. + * - ``install.enable_entrypoints`` + - (none) + - True to generate global commands for installed packages (such as + ``pip.exe``). These are defined by the packages themselves. + If set to false, only the Python interpreter has global commands created. + By default, true. You should run ``py install --refresh`` after changing + this setting. + * - ``list.format`` - ``PYTHON_MANAGER_LIST_FORMAT`` - Specify the default format used by the ``py list`` command. By default, ``table``. + * - ``install_dir`` + - (none) + - Specify the root directory that runtimes will be installed into. + If you change this setting, previously installed runtimes will not be + usable unless you move them to the new location. + + * - ``global_dir`` + - (none) + - Specify the directory where global commands (such as ``python3.14.exe`` + and ``pip.exe``) are stored. + This directory should be added to your :envvar:`PATH` to make the + commands available from your terminal. + + * - ``download_dir`` + - (none) + - Specify the directory where downloaded files are stored. + This directory is a temporary cache, and can be cleaned up from time to + time. + + Dotted names should be nested inside JSON objects, for example, ``list.format`` would be specified as ``{"list": {"format": "table"}}``. @@ -524,12 +573,9 @@ configuration option. The behaviour of shebangs in the Python install manager is subtly different from the previous ``py.exe`` launcher, and the old configuration options no longer apply. If you are specifically reliant on the old behaviour or - configuration, we recommend keeping the legacy launcher. It may be - `downloaded independently `_ - and installed on its own. The legacy launcher's ``py`` command will override - PyManager's one, and you will need to use ``pymanager`` commands for - installing and uninstalling. - + configuration, we recommend installing the `legacy launcher`_. The legacy + launcher's ``py`` command will override PyManager's one by default, and you + will need to use ``pymanager`` commands for installing and uninstalling. .. _Add-AppxPackage: https://learn.microsoft.com/powershell/module/appx/add-appxpackage @@ -541,7 +587,7 @@ configuration option. .. _pymanager-advancedinstall: -Advanced Installation +Advanced installation --------------------- For situations where an MSIX cannot be installed, such as some older @@ -635,7 +681,7 @@ the Store package in this way. .. _pymanager-admin-config: -Administrative Configuration +Administrative configuration ---------------------------- There are a number of options that may be useful for administrators to override @@ -706,6 +752,14 @@ directory containing the configuration file that specified them. (e.g. ``"pep514,start"``). Disabled shortcuts are not reactivated by ``enable_shortcut_kinds``. + * - ``install.hard_link_entrypoints`` + - True to use hard links for global shortcuts to save disk space. If false, + each shortcut executable is copied instead. After changing this setting, + you must run ``py install --refresh --force`` to update existing + commands. + By default, true. Disabling this may be necessary for troubleshooting or + systems that have issues with file links. + * - ``pep514_root`` - Registry location to read and write PEP 514 entries into. By default, :file:`HKEY_CURRENT_USER\\Software\\Python`. @@ -724,19 +778,22 @@ directory containing the configuration file that specified them. - True to suppress visible warnings when a shebang launches an application other than a Python runtime. + * - ``source_settings`` + - A mapping from source URL to settings specific to that index. + When multiple configuration files include this section, URL settings are + added or overwritten, but individual settings are not merged. + These settings are currently only for :ref:`index signatures + `. + + .. _install-freethreaded-windows: -Installing Free-threaded Binaries +Installing free-threaded binaries --------------------------------- -.. versionadded:: 3.13 (Experimental) +.. versionadded:: 3.13 -.. note:: - - Everything described in this section is considered experimental, - and should be expected to change in future releases. - -Pre-built distributions of the experimental free-threaded build are available +Pre-built distributions of the free-threaded build are available by installing tags with the ``t`` suffix. .. code:: @@ -750,6 +807,101 @@ installed, then ``python`` will launch this one. Otherwise, you will need to use ``py -V:3.14t ...`` or, if you have added the global aliases directory to your :envvar:`PATH` environment variable, the ``python3.14t.exe`` commands. + +.. _pymanager-index-signatures: + +Index signatures +---------------- + +.. versionadded:: 26.2 + +Index files may be signed to detect tampering. A signature is a catalog file +at the same URL as the index with ``.cat`` added to the filename. The catalog +file should contain the hash of its matching index file, and should be signed +with a valid Authenticode signature. This allows standard tooling (on Windows) +to generate a signature, and any certificate may be used as long as the client +operating system already trusts its certification authority (root CA). + +Index signatures are only downloaded and checked when the local configuration's +``source_settings`` section includes the index URL and ``requires_signature`` is +true, or the index JSON contains ``requires_signature`` set to true. When the +setting exists in local configuration, even when false, settings in the index +are ignored. + +As well as requiring a valid signature, the ``required_root_subject`` and +``required_publisher_subject`` settings can further restrict acceptable +signatures based on the certificate Subject fields. Any attribute specified in +the configuration must match the attribute in the certificate (additional +attributes in the certificate are ignored). Typical attributes are ``CN=`` for +the common name, ``O=`` for the organizational unit, and ``C=`` for the +publisher's country. + +Finally, the ``required_publisher_eku`` setting allows requiring that a specific +Enhanced Key Usage (EKU) has been assigned to the publisher certificate. For +example, the EKU ``1.3.6.1.5.5.7.3.3`` indicates that the certificate was +intended for code signing (as opposed to server or client authentication). +In combination with a specific root CA, this provides another mechanism to +verify a legitimate signature. + +This is an example ``source_settings`` section from a configuration file. In +this case, the publisher of the feed is uniquely identified by the combination +of the Microsoft Identity Verification root and the EKU assigned by that root. +The signature for this case would be found at +``https://www.python.org/ftp/python/index-windows.json.cat``. + +.. code:: json5 + + { + "source_settings": { + "https://www.python.org/ftp/python/index-windows.json": { + "requires_signature": true, + "required_root_subject": "CN=Microsoft Identity Verification Root Certificate Authority 2020", + "required_publisher_subject": "CN=Python Software Foundation", + "required_publisher_eku": "1.3.6.1.4.1.311.97.608394634.79987812.305991749.578777327" + } + } + } + +The same settings could be specified in the ``index.json`` file instead. In this +case, the root and EKU are omitted, meaning that the signature must be valid and +have a specific common name in the publisher's certificate, but no other checks +are used. + +.. code:: json5 + + { + "requires_signature": true, + "required_publisher_subject": "CN=Python Software Foundation", + "versions": [ + // ... + ] + } + +When settings from inside a feed are used, the user is notified and the settings +are shown in the log file or verbose output. It is recommended to copy these +settings into a local configuration file for feeds that will be used frequently, +so that unauthorised modifications to the feed cannot disable verification. + +It is not possible to override the location of the signature file in the feed or +through a configuration file. Administrators can provide their own +``source_settings`` in a mandatory configuration file (see +:ref:`pymanager-admin-config`). + +If signature validation fails, you will be notified and prompted to continue. +When interactive confirmation is not allowed (for example, because ``--yes`` was +specified), it will always abort. To use a feed with invalid configuration in +this scenario, you must provide a configuration file that disables signature +checking for that feed. + +.. code:: json5 + + "source_settings": { + "https://www.example.com/feed-with-invalid-signature.json": { + "requires_signature": false + } + } + + .. _pymanager-troubleshoot: Troubleshooting @@ -757,7 +909,7 @@ Troubleshooting If your Python install manager does not seem to be working correctly, please work through these tests and fixes to see if it helps. If not, please report an -issue at `our bug tracker `_, +issue at `our bug tracker `_, including any relevant log files (written to your :file:`%TEMP%` directory by default). @@ -786,6 +938,12 @@ default). * - - Check that the ``py`` and ``pymanager`` commands work. + * - + - Ensure your :envvar:`PATH` variable contains the entry for + ``%UserProfile%\AppData\Local\Microsoft\WindowsApps``. + The operating system includes this entry once by default, after other + user paths. If removed, shortcuts will not be found. + * - ``py`` gives me a "command not found" error when I type it in my terminal. - Did you :ref:`install the Python install manager `? @@ -796,6 +954,12 @@ default). The "Python (default windowed)" and "Python install manager" commands may also need refreshing. + * - + - Ensure your :envvar:`PATH` variable contains the entry for + ``%UserProfile%\AppData\Local\Microsoft\WindowsApps``. + The operating system includes this entry once by default, after other + user paths. If removed, shortcuts will not be found. + * - ``py`` gives me a "can't open file" error when I type commands in my terminal. - This usually means you have the legacy launcher installed and @@ -826,7 +990,7 @@ default). - Prerelease and experimental installs that are not managed by the Python install manager may be chosen ahead of stable releases. Configure your default tag or uninstall the prerelease runtime - and reinstall using ``py install``. + and reinstall it using ``py install``. * - ``pythonw`` or ``pyw`` don't launch the same runtime as ``python`` or ``py`` - Click Start, open "Manage app execution aliases", and check that your @@ -838,13 +1002,48 @@ default). * - - The package may be available but missing the generated executable. - We recommend using the ``python -m pip`` command instead, - or alternatively the ``python -m pip install --force pip`` command - will recreate the executables and show you the path to - add to :envvar:`PATH`. - These scripts are separated for each runtime, and so you may need to - add multiple paths. + We recommend using the ``python -m pip`` command instead. + Running ``py install --refresh`` and ensuring that the global shortcuts + directory is on :envvar:`PATH` (it will be shown in the command output if + it is not) should make commands such as ``pip`` (and other installed + packages) available. + * - I installed a package with ``pip`` but its command is not found. + - Have you activated a virtual environment? + Run the ``.venv\Scripts\activate`` script in your terminal to activate. + + * - + - New packages do not automatically have global shortcuts created by the + Python install manager. Similarly, uninstalled packages do not have their + shortcuts removed. + Run ``py install --refresh`` to update the global shortcuts for newly + installed packages. + + * - Typing ``script-name.py`` in the terminal opens in a new window. + - This is a known limitation of the operating system. Either specify ``py`` + before the script name, create a batch file containing ``@py "%~dpn0.py" %*`` + with the same name as the script, or install the `legacy launcher`_ + and select it as the association for scripts. + + * - Drag-dropping files onto a script doesn't work + - This is a known limitation of the operating system. It is supported with + the `legacy launcher`_, or with the Python install manager when installed + from the MSI. + + * - I have installed the Python install manager multiple times. + - It is possible to install from the Store or WinGet, from the MSIX on + the Python website, and from the MSI, all at once. + They are all compatible and will share configuration and runtimes. + + * - + - See the earlier :ref:`pymanager-advancedinstall` section for ways to + uninstall the install manager other than the typical Installed Apps + (Add and Remove Programs) settings page. + + * - My old ``py.ini`` settings no longer work. + - The new Python install manager no longer supports this configuration file + or its settings, and so it will be ignored. + See :ref:`pymanager-config` for information about configuration settings. .. _windows-embeddable: @@ -862,7 +1061,7 @@ To install an embedded distribution, we recommend using ``py install`` with the .. code:: - $> py install 3.14-embed --target=runtime + $> py install 3.14-embed --target= When extracted, the embedded distribution is (almost) fully isolated from the user's system, including environment variables, system registry settings, and @@ -885,7 +1084,7 @@ versions before providing updates to users. The two recommended use cases for this distribution are described below. -Python Application +Python application ------------------ An application written in Python does not necessarily require users to be aware @@ -989,12 +1188,7 @@ for the 64-bit version, `www.nuget.org/packages/pythonx86 Free-threaded packages ---------------------- -.. versionadded:: 3.13 (Experimental) - -.. note:: - - Everything described in this section is considered experimental, - and should be expected to change in future releases. +.. versionadded:: 3.13 Packages containing free-threaded binaries are named `python-freethreaded `_ @@ -1046,7 +1240,7 @@ please install Python 3.12. .. _max-path: -Removing the MAX_PATH Limitation +Removing the MAX_PATH limitation ================================ Windows historically has limited path lengths to 260 characters. This meant that @@ -1070,7 +1264,7 @@ UTF-8 mode ========== .. versionadded:: 3.7 -.. versionchanged:: next +.. versionchanged:: 3.15 Python UTF-8 mode is now enabled by default (:pep:`686`). @@ -1332,7 +1526,7 @@ installation". In this case: * Shortcuts are available for all users -Removing the MAX_PATH Limitation +Removing the MAX_PATH limitation -------------------------------- Windows historically has limited path lengths to 260 characters. This meant that @@ -1355,7 +1549,7 @@ After changing the above option, no further configuration is required. .. _install-quiet-option: -Installing Without UI +Installing without UI --------------------- All of the options available in the installer UI can also be specified from the @@ -1504,7 +1698,7 @@ example file sets the same options as the previous example: .. _install-layout-option: -Installing Without Downloading +Installing without downloading ------------------------------ As some features of Python are not included in the initial installer download, @@ -1545,15 +1739,10 @@ settings and replace any that have been removed or modified. :ref:`launcher`, which has its own entry in Programs and Features. -Installing Free-threaded Binaries +Installing free-threaded binaries --------------------------------- -.. versionadded:: 3.13 (Experimental) - -.. note:: - - Everything described in this section is considered experimental, - and should be expected to change in future releases. +.. versionadded:: 3.13 To install pre-built binaries with free-threading enabled (see :pep:`703`), you should select "Customize installation". The second page of options includes the @@ -1585,7 +1774,7 @@ builds. Free-threaded binaries are also available :ref:`on nuget.org `. -Python Launcher for Windows (Deprecated) +Python launcher for Windows (deprecated) ======================================== .. deprecated:: 3.14 @@ -1737,7 +1926,7 @@ have the script specify the version which should be used. The key benefit of this is that a single launcher can support multiple Python versions at the same time depending on the contents of the first line. -Shebang Lines +Shebang lines ------------- If the first line of a script file starts with ``#!``, it is known as a @@ -1949,7 +2138,7 @@ should allow you to see what versions of Python were located, why a particular version was chosen and the exact command-line used to execute the target Python. It is primarily intended for testing and debugging. -Dry Run +Dry run ------- If an environment variable :envvar:`!PYLAUNCHER_DRYRUN` is set (to any value), diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index b7e4e73f4ce..43ab19037d2 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -66,7 +66,7 @@ Here's a simple example:: The union and intersection of sets can be computed with the :meth:`~frozenset.union` and :meth:`~frozenset.intersection` methods; an alternative notation uses the bitwise operators ``&`` and ``|``. Mutable sets also have in-place versions of these methods, -:meth:`!union_update` and :meth:`~frozenset.intersection_update`. :: +:meth:`!union_update` and :meth:`~set.intersection_update`. :: >>> S1 = sets.Set([1,2,3]) >>> S2 = sets.Set([4,5,6]) @@ -87,7 +87,7 @@ It's also possible to take the symmetric difference of two sets. This is the set of all elements in the union that aren't in the intersection. Another way of putting it is that the symmetric difference contains all elements that are in exactly one set. Again, there's an alternative notation (``^``), and an -in-place version with the ungainly name :meth:`~frozenset.symmetric_difference_update`. :: +in-place version with the ungainly name :meth:`~set.symmetric_difference_update`. :: >>> S1 = sets.Set([1,2,3,4]) >>> S2 = sets.Set([3,4,5,6]) @@ -1698,8 +1698,8 @@ current local date. Once created, instances of the date/time classes are all immutable. There are a number of methods for producing formatted strings from objects:: - >>> import datetime - >>> now = datetime.datetime.now() + >>> import datetime as dt + >>> now = dt.datetime.now() >>> now.isoformat() '2002-12-30T21:27:03.994956' >>> now.ctime() # Only available on date, datetime @@ -1710,10 +1710,10 @@ number of methods for producing formatted strings from objects:: The :meth:`~datetime.datetime.replace` method allows modifying one or more fields of a :class:`~datetime.date` or :class:`~datetime.datetime` instance, returning a new instance:: - >>> d = datetime.datetime.now() + >>> d = dt.datetime.now() >>> d datetime.datetime(2002, 12, 30, 22, 15, 38, 827738) - >>> d.replace(year=2001, hour = 12) + >>> d.replace(year=2001, hour=12) datetime.datetime(2001, 12, 30, 12, 15, 38, 827738) >>> diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 3430ac8668e..03e612fb651 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -1277,12 +1277,12 @@ complete list of changes, or look through the SVN logs for all the details. with the new ':keyword:`with`' statement. See section :ref:`contextlibmod` for more about this module. -* New module: The :mod:`cProfile` module is a C implementation of the existing - :mod:`profile` module that has much lower overhead. The module's interface is - the same as :mod:`profile`: you run ``cProfile.run('main()')`` to profile a +* New module: The :mod:`!cProfile` module is a C implementation of the existing + :mod:`!profile` module that has much lower overhead. The module's interface is + the same as :mod:`!profile`: you run ``cProfile.run('main()')`` to profile a function, can save profile data to a file, etc. It's not yet known if the Hotshot profiler, which is also written in C but doesn't match the - :mod:`profile` module's interface, will continue to be maintained in future + :mod:`!profile` module's interface, will continue to be maintained in future versions of Python. (Contributed by Armin Rigo.) Also, the :mod:`pstats` module for analyzing the data measured by the profiler @@ -1313,10 +1313,10 @@ complete list of changes, or look through the SVN logs for all the details. by Josh Spoerri. It uses the same format characters as :func:`time.strptime` and :func:`time.strftime`:: - from datetime import datetime + import datetime as dt - ts = datetime.strptime('10:13:15 2006-03-07', - '%H:%M:%S %Y-%m-%d') + ts = dt.datetime.strptime('10:13:15 2006-03-07', + '%H:%M:%S %Y-%m-%d') * The :meth:`SequenceMatcher.get_matching_blocks` method in the :mod:`difflib` module now guarantees to return a minimal list of blocks describing matching @@ -2169,9 +2169,9 @@ Changes to Python's build process and to the C API include: * Two new macros can be used to indicate C functions that are local to the current file so that a faster calling convention can be used. - ``Py_LOCAL(type)`` declares the function as returning a value of the + :c:macro:`Py_LOCAL` declares the function as returning a value of the specified *type* and uses a fast-calling qualifier. - ``Py_LOCAL_INLINE(type)`` does the same thing and also requests the + :c:macro:`Py_LOCAL_INLINE` does the same thing and also requests the function be inlined. If macro :c:macro:`!PY_LOCAL_AGGRESSIVE` is defined before :file:`python.h` is included, a set of more aggressive optimizations are enabled for the module; you should benchmark the results to find out if these diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index cbab2b57cbb..1215601a09d 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -56,7 +56,7 @@ Python 2.6 incorporates new features and syntax from 3.0 while remaining compatible with existing code by not removing older features or syntax. When it's not possible to do that, Python 2.6 tries to do what it can, adding compatibility functions in a -:mod:`future_builtins` module and a :option:`!-3` switch to warn about +:mod:`!future_builtins` module and a :option:`!-3` switch to warn about usages that will become unsupported in 3.0. Some significant new packages have been added to the standard library, @@ -109,7 +109,7 @@ are: Python 3.0 adds several new built-in functions and changes the semantics of some existing builtins. Functions that are new in 3.0 such as :func:`bin` have simply been added to Python 2.6, but existing -builtins haven't been changed; instead, the :mod:`future_builtins` +builtins haven't been changed; instead, the :mod:`!future_builtins` module has versions with the new 3.0 semantics. Code written to be compatible with 3.0 can do ``from future_builtins import hex, map`` as necessary. @@ -118,7 +118,7 @@ A new command-line switch, :option:`!-3`, enables warnings about features that will be removed in Python 3.0. You can run code with this switch to see how much work will be necessary to port code to 3.0. The value of this switch is available -to Python code as the boolean variable :data:`sys.py3kwarning`, +to Python code as the boolean variable :data:`!sys.py3kwarning`, and to C extension code as :c:data:`!Py_Py3kWarningFlag`. .. seealso:: @@ -307,9 +307,9 @@ The :mod:`threading` module's locks and condition variables also support the The lock is acquired before the block is executed and always released once the block is complete. -The :func:`localcontext` function in the :mod:`decimal` module makes it easy -to save and restore the current decimal context, which encapsulates the desired -precision and rounding characteristics for computations:: +The :func:`~decimal.localcontext` function in the :mod:`decimal` module makes +it easy to save and restore the current decimal context, which encapsulates +the desired precision and rounding characteristics for computations:: from decimal import Decimal, Context, localcontext @@ -337,12 +337,12 @@ underlying implementation and should keep reading. A high-level explanation of the context management protocol is: * The expression is evaluated and should result in an object called a "context - manager". The context manager must have :meth:`~object.__enter__` and :meth:`~object.__exit__` - methods. + manager". The context manager must have :meth:`~object.__enter__` and + :meth:`~object.__exit__` methods. -* The context manager's :meth:`~object.__enter__` method is called. The value returned - is assigned to *VAR*. If no ``as VAR`` clause is present, the value is simply - discarded. +* The context manager's :meth:`~object.__enter__` method is called. The value + returned is assigned to *VAR*. If no ``as VAR`` clause is present, the + value is simply discarded. * The code in *BLOCK* is executed. @@ -378,7 +378,7 @@ be to let the user write code like this:: The transaction should be committed if the code in the block runs flawlessly or rolled back if there's an exception. Here's the basic interface for -:class:`DatabaseConnection` that I'll assume:: +:class:`!DatabaseConnection` that I'll assume:: class DatabaseConnection: # Database interface @@ -431,14 +431,15 @@ The contextlib module The :mod:`contextlib` module provides some functions and a decorator that are useful when writing objects for use with the ':keyword:`with`' statement. -The decorator is called :func:`contextmanager`, and lets you write a single -generator function instead of defining a new class. The generator should yield -exactly one value. The code up to the :keyword:`yield` will be executed as the -:meth:`~object.__enter__` method, and the value yielded will be the method's return -value that will get bound to the variable in the ':keyword:`with`' statement's -:keyword:`!as` clause, if any. The code after the :keyword:`!yield` will be -executed in the :meth:`~object.__exit__` method. Any exception raised in the block will -be raised by the :keyword:`!yield` statement. +The decorator is called :func:`~contextlib.contextmanager`, and lets you write +a single generator function instead of defining a new class. The generator +should yield exactly one value. The code up to the :keyword:`yield` will be +executed as the :meth:`~object.__enter__` method, and the value yielded will +be the method's return value that will get bound to the variable in the +':keyword:`with`' statement's :keyword:`!as` clause, if any. The code after +the :keyword:`!yield` will be executed in the :meth:`~object.__exit__` method. +Any exception raised in the block will be raised by the :keyword:`!yield` +statement. Using this decorator, our database example from the previous section could be written as:: @@ -469,7 +470,7 @@ statement both starts a database transaction and acquires a thread lock:: with nested (db_transaction(db), lock) as (cursor, locked): ... -Finally, the :func:`closing` function returns its argument so that it can be +Finally, the :func:`~contextlib.closing` function returns its argument so that it can be bound to a variable, and calls the argument's ``.close()`` method at the end of the block. :: @@ -538,7 +539,7 @@ If you don't like the default directory, it can be overridden by an environment variable. :envvar:`PYTHONUSERBASE` sets the root directory used for all Python versions supporting this feature. On Windows, the directory for application-specific data can be changed by -setting the :envvar:`APPDATA` environment variable. You can also +setting the :envvar:`!APPDATA` environment variable. You can also modify the :file:`site.py` file for your Python installation. The feature can be disabled entirely by running Python with the @@ -568,11 +569,12 @@ The :mod:`multiprocessing` module started out as an exact emulation of the :mod:`threading` module using processes instead of threads. That goal was discarded along the path to Python 2.6, but the general approach of the module is still similar. The fundamental class -is the :class:`Process`, which is passed a callable object and -a collection of arguments. The :meth:`start` method +is the :class:`~multiprocessing.Process`, which is passed a callable object and +a collection of arguments. The :meth:`~multiprocessing.Process.start` method sets the callable running in a subprocess, after which you can call -the :meth:`is_alive` method to check whether the subprocess is still running -and the :meth:`join` method to wait for the process to exit. +the :meth:`~multiprocessing.Process.is_alive` method to check whether the +subprocess is still running and the :meth:`~multiprocessing.Process.join` +method to wait for the process to exit. Here's a simple example where the subprocess will calculate a factorial. The function doing the calculation is written strangely so @@ -619,13 +621,16 @@ the object to communicate. (If the parent were to change the value of the global variable, the child's value would be unaffected, and vice versa.) -Two other classes, :class:`Pool` and :class:`Manager`, provide -higher-level interfaces. :class:`Pool` will create a fixed number of -worker processes, and requests can then be distributed to the workers -by calling :meth:`apply` or :meth:`apply_async` to add a single request, -and :meth:`map` or :meth:`map_async` to add a number of -requests. The following code uses a :class:`Pool` to spread requests -across 5 worker processes and retrieve a list of results:: +Two other classes, :class:`~multiprocessing.pool.Pool` and +:class:`~multiprocessing.Manager`, provide higher-level interfaces. +:class:`~multiprocessing.pool.Pool` will create a fixed number of worker +processes, and requests can then be distributed to the workers by calling +:meth:`~multiprocessing.pool.Pool.apply` or +:meth:`~multiprocessing.pool.Pool.apply_async` to add a single request, and +:meth:`~multiprocessing.pool.Pool.map` or +:meth:`~multiprocessing.pool.Pool.map_async` to add a number of +requests. The following code uses a :class:`~multiprocessing.pool.Pool` to +spread requests across 5 worker processes and retrieve a list of results:: from multiprocessing import Pool @@ -646,15 +651,18 @@ This produces the following output:: 33452526613163807108170062053440751665152000000000 ... -The other high-level interface, the :class:`Manager` class, creates a -separate server process that can hold master copies of Python data +The other high-level interface, the :class:`~multiprocessing.Manager` class, +creates a separate server process that can hold master copies of Python data structures. Other processes can then access and modify these data structures using proxy objects. The following example creates a shared dictionary by calling the :meth:`dict` method; the worker processes then insert values into the dictionary. (Locking is not done for you automatically, which doesn't matter in this example. -:class:`Manager`'s methods also include :meth:`Lock`, :meth:`RLock`, -and :meth:`Semaphore` to create shared locks.) +:class:`~multiprocessing.Manager`'s methods also include +:meth:`~multiprocessing.managers.SyncManager.Lock`, +:meth:`~multiprocessing.managers.SyncManager.RLock`, +and :meth:`~multiprocessing.managers.SyncManager.Semaphore` to create +shared locks.) :: @@ -824,7 +832,7 @@ documentation for a :ref:`complete list `; here's a sample: format, followed by a percent sign. ===== ======================================================================== -Classes and types can define a :meth:`__format__` method to control how they're +Classes and types can define a :meth:`~object.__format__` method to control how they're formatted. It receives a single argument, the format specifier:: def __format__(self, format_spec): @@ -834,7 +842,7 @@ formatted. It receives a single argument, the format specifier:: return str(self) There's also a :func:`format` builtin that will format a single -value. It calls the type's :meth:`__format__` method with the +value. It calls the type's :meth:`~object.__format__` method with the provided specifier:: >>> format(75.6564, '.2f') @@ -1029,56 +1037,58 @@ PEP 3116: New I/O Library Python's built-in file objects support a number of methods, but file-like objects don't necessarily support all of them. Objects that -imitate files usually support :meth:`read` and :meth:`write`, but they -may not support :meth:`readline`, for example. Python 3.0 introduces -a layered I/O library in the :mod:`io` module that separates buffering -and text-handling features from the fundamental read and write -operations. +imitate files usually support :meth:`!read` and +:meth:`!write`, but they may not support :meth:`!readline`, +for example. Python 3.0 introduces a layered I/O library in the :mod:`io` +module that separates buffering and text-handling features from the +fundamental read and write operations. There are three levels of abstract base classes provided by the :mod:`io` module: -* :class:`RawIOBase` defines raw I/O operations: :meth:`read`, - :meth:`readinto`, - :meth:`write`, :meth:`seek`, :meth:`tell`, :meth:`truncate`, - and :meth:`close`. +* :class:`~io.RawIOBase` defines raw I/O operations: :meth:`~io.RawIOBase.read`, + :meth:`~io.RawIOBase.readinto`, :meth:`~io.RawIOBase.write`, + :meth:`~io.IOBase.seek`, :meth:`~io.IOBase.tell`, :meth:`~io.IOBase.truncate`, + and :meth:`~io.IOBase.close`. Most of the methods of this class will often map to a single system call. - There are also :meth:`readable`, :meth:`writable`, and :meth:`seekable` - methods for determining what operations a given object will allow. + There are also :meth:`~io.IOBase.readable`, :meth:`~io.IOBase.writable`, + and :meth:`~io.IOBase.seekable` methods for determining what operations a + given object will allow. Python 3.0 has concrete implementations of this class for files and sockets, but Python 2.6 hasn't restructured its file and socket objects in this way. -* :class:`BufferedIOBase` is an abstract base class that +* :class:`~io.BufferedIOBase` is an abstract base class that buffers data in memory to reduce the number of system calls used, making I/O processing more efficient. - It supports all of the methods of :class:`RawIOBase`, - and adds a :attr:`raw` attribute holding the underlying raw object. + It supports all of the methods of :class:`~io.RawIOBase`, + and adds a :attr:`~io.BufferedIOBase.raw` attribute holding the underlying + raw object. There are five concrete classes implementing this ABC. - :class:`BufferedWriter` and :class:`BufferedReader` are for objects - that support write-only or read-only usage that have a :meth:`seek` - method for random access. :class:`BufferedRandom` objects support + :class:`~io.BufferedWriter` and :class:`~io.BufferedReader` are for objects + that support write-only or read-only usage that have a :meth:`~io.IOBase.seek` + method for random access. :class:`~io.BufferedRandom` objects support read and write access upon the same underlying stream, and - :class:`BufferedRWPair` is for objects such as TTYs that have both + :class:`~io.BufferedRWPair` is for objects such as TTYs that have both read and write operations acting upon unconnected streams of data. - The :class:`BytesIO` class supports reading, writing, and seeking + The :class:`~io.BytesIO` class supports reading, writing, and seeking over an in-memory buffer. .. index:: single: universal newlines; What's new -* :class:`TextIOBase`: Provides functions for reading and writing +* :class:`~io.TextIOBase`: Provides functions for reading and writing strings (remember, strings will be Unicode in Python 3.0), - and supporting :term:`universal newlines`. :class:`TextIOBase` defines + and supporting :term:`universal newlines`. :class:`~io.TextIOBase` defines the :meth:`readline` method and supports iteration upon objects. - There are two concrete implementations. :class:`TextIOWrapper` + There are two concrete implementations. :class:`~io.TextIOWrapper` wraps a buffered I/O object, supporting all of the methods for - text I/O and adding a :attr:`buffer` attribute for access - to the underlying object. :class:`StringIO` simply buffers + text I/O and adding a :attr:`~io.TextIOBase.buffer` attribute for access + to the underlying object. :class:`~io.StringIO` simply buffers everything in memory without ever writing anything to disk. (In Python 2.6, :class:`io.StringIO` is implemented in @@ -1162,7 +1172,7 @@ Some object-oriented languages such as Java support interfaces, declaring that a class has a given set of methods or supports a given access protocol. Abstract Base Classes (or ABCs) are an equivalent feature for Python. The ABC support consists of an :mod:`abc` module -containing a metaclass called :class:`ABCMeta`, special handling of +containing a metaclass called :class:`~abc.ABCMeta`, special handling of this metaclass by the :func:`isinstance` and :func:`issubclass` builtins, and a collection of basic ABCs that the Python developers think will be widely useful. Future versions of Python will probably @@ -1172,17 +1182,17 @@ Let's say you have a particular class and wish to know whether it supports dictionary-style access. The phrase "dictionary-style" is vague, however. It probably means that accessing items with ``obj[1]`` works. Does it imply that setting items with ``obj[2] = value`` works? -Or that the object will have :meth:`keys`, :meth:`values`, and :meth:`items` -methods? What about the iterative variants such as :meth:`iterkeys`? :meth:`copy` -and :meth:`update`? Iterating over the object with :func:`iter`? +Or that the object will have :meth:`!keys`, :meth:`!values`, and :meth:`!items` +methods? What about the iterative variants such as :meth:`!iterkeys`? +:meth:`!copy`and :meth:`!update`? Iterating over the object with :func:`!iter`? The Python 2.6 :mod:`collections` module includes a number of different ABCs that represent these distinctions. :class:`Iterable` -indicates that a class defines :meth:`__iter__`, and -:class:`Container` means the class defines a :meth:`__contains__` +indicates that a class defines :meth:`~object.__iter__`, and +:class:`Container` means the class defines a :meth:`~object.__contains__` method and therefore supports ``x in y`` expressions. The basic dictionary interface of getting items, setting items, and -:meth:`keys`, :meth:`values`, and :meth:`items`, is defined by the +:meth:`!keys`, :meth:`!values`, and :meth:`!items`, is defined by the :class:`MutableMapping` ABC. You can derive your own classes from a particular ABC @@ -1196,7 +1206,7 @@ to indicate they support that ABC's interface:: Alternatively, you could write the class without deriving from the desired ABC and instead register the class by -calling the ABC's :meth:`register` method:: +calling the ABC's :meth:`~abc.ABCMeta.register` method:: import collections @@ -1206,10 +1216,10 @@ calling the ABC's :meth:`register` method:: collections.MutableMapping.register(Storage) For classes that you write, deriving from the ABC is probably clearer. -The :meth:`register` method is useful when you've written a new +The :meth:`~abc.ABCMeta.register` method is useful when you've written a new ABC that can describe an existing type or class, or if you want to declare that some third-party class implements an ABC. -For example, if you defined a :class:`PrintableType` ABC, +For example, if you defined a :class:`!PrintableType` ABC, it's legal to do:: # Register Python's types @@ -1256,16 +1266,16 @@ metaclass in a class definition:: ... -In the :class:`Drawable` ABC above, the :meth:`draw_doubled` method +In the :class:`!Drawable` ABC above, the :meth:`!draw_doubled` method renders the object at twice its size and can be implemented in terms -of other methods described in :class:`Drawable`. Classes implementing +of other methods described in :class:`!Drawable`. Classes implementing this ABC therefore don't need to provide their own implementation -of :meth:`draw_doubled`, though they can do so. An implementation -of :meth:`draw` is necessary, though; the ABC can't provide +of :meth:`!draw_doubled`, though they can do so. An implementation +of :meth:`!draw` is necessary, though; the ABC can't provide a useful generic implementation. -You can apply the ``@abstractmethod`` decorator to methods such as -:meth:`draw` that must be implemented; Python will then raise an +You can apply the :deco:`~abc.abstractmethod` decorator to methods such as +:meth:`!draw` that must be implemented; Python will then raise an exception for classes that don't define the method. Note that the exception is only raised when you actually try to create an instance of a subclass lacking the method:: @@ -1289,7 +1299,7 @@ Abstract data attributes can be declared using the def readonly(self): return self._x -Subclasses must then define a :meth:`readonly` property. +Subclasses must then define a ``readonly`` property. .. seealso:: @@ -2739,13 +2749,13 @@ numbers. .. ====================================================================== -The :mod:`future_builtins` module +The :mod:`!future_builtins` module -------------------------------------- Python 3.0 makes many changes to the repertoire of built-in functions, and most of the changes can't be introduced in the Python 2.x series because they would break compatibility. -The :mod:`future_builtins` module provides versions +The :mod:`!future_builtins` module provides versions of these built-in functions that can be imported when writing 3.0-compatible code. @@ -2812,10 +2822,10 @@ Using the module is simple:: import sys import plistlib - import datetime + import datetime as dt # Create data structure - data_struct = dict(lastAccessed=datetime.datetime.now(), + data_struct = dict(lastAccessed=dt.datetime.now(), version=1, categories=('Personal','Shared','Private')) diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index bcc5a3d5690..7296296d144 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -858,8 +858,8 @@ Some smaller changes made to the core Python language are: .. XXX bytearray doesn't seem to be documented -* When using :class:`@classmethod ` and - :class:`@staticmethod ` to wrap +* When using :deco:`classmethod` and + :deco:`staticmethod` to wrap methods as class or static methods, the wrapper object now exposes the wrapped function as their :attr:`~method.__func__` attribute. @@ -2181,14 +2181,14 @@ Changes to Python's build process and to the C API include: discussed in :issue:`5753`, and fixed by Antoine Pitrou. * New macros: the Python header files now define the following macros: - :c:macro:`!Py_ISALNUM`, - :c:macro:`!Py_ISALPHA`, - :c:macro:`!Py_ISDIGIT`, - :c:macro:`!Py_ISLOWER`, - :c:macro:`!Py_ISSPACE`, - :c:macro:`!Py_ISUPPER`, - :c:macro:`!Py_ISXDIGIT`, - :c:macro:`!Py_TOLOWER`, and :c:macro:`!Py_TOUPPER`. + :c:macro:`Py_ISALNUM`, + :c:macro:`Py_ISALPHA`, + :c:macro:`Py_ISDIGIT`, + :c:macro:`Py_ISLOWER`, + :c:macro:`Py_ISSPACE`, + :c:macro:`Py_ISUPPER`, + :c:macro:`Py_ISXDIGIT`, + :c:macro:`Py_TOLOWER`, and :c:macro:`Py_TOUPPER`. All of these functions are analogous to the C standard macros for classifying characters, but ignore the current locale setting, because in @@ -2196,8 +2196,6 @@ Changes to Python's build process and to the C API include: locale-independent way. (Added by Eric Smith; :issue:`5793`.) - .. XXX these macros don't seem to be described in the c-api docs. - * Removed function: :c:func:`!PyEval_CallObject` is now only available as a macro. A function version was being kept around to preserve ABI linking compatibility, but that was in 1997; it can certainly be diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 8db57f6f22f..8a78dbd9038 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -402,7 +402,7 @@ Tracing events, with the correct line number, are generated for all lines of cod The :attr:`~frame.f_lineno` attribute of frame objects will always contain the expected line number. -The :attr:`~codeobject.co_lnotab` attribute of +The :attr:`!codeobject.co_lnotab` attribute of :ref:`code objects ` is deprecated and will be removed in 3.12. Code that needs to convert from offset to line number should use the new @@ -847,8 +847,8 @@ Other Language Changes respectively. (Contributed by Joshua Bronson, Daniel Pope, and Justin Wang in :issue:`31861`.) -* Static methods (:func:`@staticmethod `) and class methods - (:func:`@classmethod `) now inherit the method attributes +* Static methods (:deco:`staticmethod`) and class methods + (:deco:`classmethod`) now inherit the method attributes (``__module__``, ``__name__``, ``__qualname__``, ``__doc__``, ``__annotations__``) and have a new ``__wrapped__`` attribute. Moreover, static methods are now callable as regular functions. @@ -1622,7 +1622,7 @@ Deprecated compatibility. Specifically, :meth:`!find_loader`/:meth:`!find_module` (superseded by :meth:`~importlib.abc.MetaPathFinder.find_spec`), - :meth:`~importlib.abc.Loader.load_module` + ``importlib.abc.Loader.load_module`` (superseded by :meth:`~importlib.abc.Loader.exec_module`), :meth:`!module_repr` (which the import system takes care of for you), the ``__package__`` attribute @@ -1652,7 +1652,7 @@ Deprecated preference for :meth:`~zipimport.zipimporter.exec_module`. (Contributed by Brett Cannon in :issue:`26131`.) -* The use of :meth:`~importlib.abc.Loader.load_module` by the import +* The use of ``importlib.abc.Loader.load_module`` by the import system now triggers an :exc:`ImportWarning` as :meth:`~importlib.abc.Loader.exec_module` is preferred. (Contributed by Brett Cannon in :issue:`26131`.) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 2dd205dd2b8..a095d887352 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2673,7 +2673,7 @@ Removed (Contributed by Victor Stinner in :issue:`45474`.) -* Exclude :c:func:`PyWeakref_GET_OBJECT` from the limited C API. It never +* Exclude :c:func:`!PyWeakref_GET_OBJECT` from the limited C API. It never worked since the :c:type:`!PyWeakReference` structure is opaque in the limited C API. (Contributed by Victor Stinner in :issue:`35134`.) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 7cfdc287b7f..df6cc98eaf1 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1191,9 +1191,23 @@ Deprecated replaced by :data:`calendar.JANUARY` and :data:`calendar.FEBRUARY`. (Contributed by Prince Roshan in :gh:`103636`.) -* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. - Prefer :class:`Sequence` or :class:`collections.abc.Buffer`. - For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. +* :mod:`collections.abc`: Deprecated :class:`collections.abc.ByteString`. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` implements + the :ref:`buffer protocol ` at runtime. For use in type + annotations, either use :class:`~collections.abc.Buffer` or a union + that explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was an + instance of :class:`!ByteString` never actually told you anything useful + about the object. Other common buffer types such as :class:`memoryview` were + also never understood as subtypes of :class:`!ByteString` (either at + runtime or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. (Contributed by Shantanu Jain in :gh:`91896`.) * :mod:`datetime`: :class:`datetime.datetime`'s :meth:`~datetime.datetime.utcnow` and @@ -1301,7 +1315,7 @@ Deprecated :class:`collections.abc.Hashable` and :class:`collections.abc.Sized` respectively, are deprecated. (:gh:`94309`.) - * :class:`!typing.ByteString`, deprecated since Python 3.9, now causes a + * :class:`typing.ByteString`, deprecated since Python 3.9, now causes a :exc:`DeprecationWarning` to be emitted when it is used. (Contributed by Alex Waygood in :gh:`91896`.) @@ -1323,7 +1337,7 @@ Deprecated it was :exc:`ImportWarning`). (Contributed by Brett Cannon in :gh:`65961`.) -* Setting :attr:`~module.__package__` or :attr:`~module.__cached__` on a +* Setting :attr:`~module.__package__` or ``__cached__`` on a module is deprecated, and will cease to be set or taken into consideration by the import system in Python 3.14. (Contributed by Brett Cannon in :gh:`65961`.) @@ -1333,7 +1347,7 @@ Deprecated ``int``, convert to int explicitly: ``~int(x)``. (Contributed by Tim Hoffmann in :gh:`103487`.) -* Accessing :attr:`~codeobject.co_lnotab` on code objects was deprecated in +* Accessing :attr:`!codeobject.co_lnotab` on code objects was deprecated in Python 3.10 via :pep:`626`, but it only got a proper :exc:`DeprecationWarning` in 3.12. May be removed in 3.15. @@ -1349,6 +1363,10 @@ Deprecated .. include:: ../deprecations/pending-removal-in-3.17.rst +.. include:: ../deprecations/pending-removal-in-3.19.rst + +.. include:: ../deprecations/pending-removal-in-3.20.rst + .. include:: ../deprecations/pending-removal-in-future.rst .. _whatsnew312-removed: diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 6545da66afc..f4489bfa1b7 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -60,7 +60,7 @@ Summary -- Release Highlights .. This section singles out the most important changes in Python 3.13. Brevity is key. -Python 3.13 is the latest stable release of the Python programming +Python 3.13 is a stable release of the Python programming language, with a mix of changes to the language, the implementation and the standard library. The biggest changes include a new `interactive interpreter @@ -1569,8 +1569,6 @@ and are now removed: * :pypi:`bcrypt`: Modern password hashing for your software and your servers. - * :pypi:`passlib`: - Comprehensive password hashing framework supporting over 30 schemes. * :pypi:`argon2-cffi`: The secure Argon2 password hashing algorithm. * :pypi:`legacycrypt`: @@ -1863,6 +1861,7 @@ New Deprecations * Deprecate the non-standard and undocumented :class:`~decimal.Decimal` format specifier ``'N'``, which is only supported in the :mod:`!decimal` module's C implementation. + Scheduled to be removed in Python 3.18. (Contributed by Serhiy Storchaka in :gh:`89902`.) * :mod:`dis`: @@ -1995,7 +1994,7 @@ New Deprecations use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. (Contributed by Alex Waygood in :gh:`105566` and :gh:`105570`.) - * Deprecate the :func:`typing.no_type_check_decorator` decorator function, + * Deprecate the :func:`!typing.no_type_check_decorator` decorator function, to be removed in Python 3.15. After eight years in the :mod:`typing` module, it has yet to be supported by any major type checker. @@ -2026,6 +2025,12 @@ New Deprecations .. include:: ../deprecations/pending-removal-in-3.17.rst +.. include:: ../deprecations/pending-removal-in-3.18.rst + +.. include:: ../deprecations/pending-removal-in-3.19.rst + +.. include:: ../deprecations/pending-removal-in-3.20.rst + .. include:: ../deprecations/pending-removal-in-future.rst CPython Bytecode Changes @@ -2246,7 +2251,7 @@ New Features (Contributed by Serhiy Storchaka in :gh:`110289`.) * Add the :c:func:`PyWeakref_GetRef` function - as an alternative to :c:func:`PyWeakref_GetObject` + as an alternative to :c:func:`!PyWeakref_GetObject` that returns a :term:`strong reference` or ``NULL`` if the referent is no longer live. (Contributed by Victor Stinner in :gh:`105927`.) @@ -2335,7 +2340,7 @@ Limited C API Changes * :c:func:`PySys_AuditTuple` * :c:func:`PyType_GetModuleByDef` - (Contributed by Victor Stinner in :gh:`85283`, :gh:`85283`, and :gh:`116936`.) + (Contributed by Victor Stinner in :gh:`85283` and :gh:`116936`.) * Python built with :option:`--with-trace-refs` (tracing references) now supports the :ref:`Limited API `. @@ -2490,7 +2495,7 @@ Deprecated C APIs * Deprecate old Python initialization functions: - * :c:func:`PySys_ResetWarnOptions`: + * :c:func:`!PySys_ResetWarnOptions`: Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. * :c:func:`!Py_GetExecPrefix`: Get :data:`sys.exec_prefix` instead. @@ -2531,8 +2536,8 @@ Deprecated C APIs are just aliases to :c:type:`!wchar_t`. (Contributed by Victor Stinner in :gh:`105156`.) -* Deprecate the :c:func:`PyWeakref_GetObject` and - :c:func:`PyWeakref_GET_OBJECT` functions, +* Deprecate the :c:func:`!PyWeakref_GetObject` and + :c:func:`!PyWeakref_GET_OBJECT` functions, which return a :term:`borrowed reference`. Replace them with the new :c:func:`PyWeakref_GetRef` function, which returns a :term:`strong reference`. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 5462c5dc425..0bb8858aea1 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1,8 +1,9 @@ + **************************** What's new in Python 3.14 **************************** -:Editor: Hugo van Kemenade +:Editors: Adam Turner and Hugo van Kemenade .. Rules for maintenance: @@ -45,458 +46,115 @@ when researching a change. This article explains the new features in Python 3.14, compared to 3.13. - +Python 3.14 was released on 7 October 2025. For full details, see the :ref:`changelog `. .. seealso:: :pep:`745` -- Python 3.14 release schedule -.. note:: - Prerelease users should be aware that this document is currently in draft - form. It will be updated substantially as Python 3.14 moves towards release, - so it's worth checking back even after reading earlier versions. - - -Summary -- release highlights +Summary -- Release highlights ============================= .. This section singles out the most important changes in Python 3.14. Brevity is key. -Python 3.14 will be the latest stable release of the Python -programming language, with a mix of changes to the language, the -implementation and the standard library. +Python 3.14 is the latest stable release of the Python programming +language, with a mix of changes to the language, the implementation, +and the standard library. +The biggest changes include :ref:`template string literals +`, +:ref:`deferred evaluation of annotations `, +and support for :ref:`subinterpreters ` in +the standard library. -The biggest changes to the implementation include template strings (:pep:`750`), -deferred evaluation of annotations (:pep:`649`), -and a new type of interpreter that uses tail calls. - -The library changes include the addition of a new :mod:`!annotationlib` module -for introspecting and wrapping annotations (:pep:`749`), -a new :mod:`!compression.zstd` module for Zstandard support (:pep:`784`), -plus syntax highlighting in the REPL, +The library changes include significantly improved capabilities for +:ref:`introspection in asyncio `, +:ref:`support for Zstandard ` via a new +:mod:`compression.zstd` module, syntax highlighting in the REPL, as well as the usual deprecations and removals, and improvements in user-friendliness and correctness. +This article doesn't attempt to provide a complete specification +of all new features, but instead gives a convenient overview. +For full details refer to the documentation, +such as the :ref:`Library Reference ` +and :ref:`Language Reference `. +To understand the complete implementation and design rationale for a change, +refer to the PEP for a particular new feature; +but note that PEPs usually are not kept up-to-date +once a feature has been fully implemented. +See `Porting to Python 3.14`_ for guidance on upgrading from +earlier versions of Python. + +-------------- + .. PEP-sized items next. -* :ref:`PEP 779: Free-threaded Python is officially supported ` -* :ref:`PEP 649 and 749: deferred evaluation of annotations ` -* :ref:`PEP 734: Multiple interpreters in the stdlib ` -* :ref:`PEP 741: Python configuration C API ` -* :ref:`PEP 750: Template strings ` -* :ref:`PEP 758: Allow except and except* expressions without parentheses ` -* :ref:`PEP 761: Discontinuation of PGP signatures ` -* :ref:`PEP 765: Disallow return/break/continue that exit a finally block ` +Interpreter improvements: + +* :pep:`649` and :pep:`749`: :ref:`Deferred evaluation of annotations + ` +* :pep:`734`: :ref:`Multiple interpreters in the standard library + ` +* :pep:`750`: :ref:`Template strings ` +* :pep:`758`: :ref:`Allow except and except* expressions without brackets + ` +* :pep:`765`: :ref:`Control flow in finally blocks + ` +* :pep:`768`: :ref:`Safe external debugger interface for CPython + ` +* :ref:`A new type of interpreter ` * :ref:`Free-threaded mode improvements ` -* :ref:`PEP 768: Safe external debugger interface for CPython ` -* :ref:`PEP 784: Adding Zstandard to the standard library ` -* :ref:`A new type of interpreter ` -* :ref:`Syntax highlighting in PyREPL `, - and color output in :ref:`unittest `, - :ref:`argparse `, - :ref:`json ` and - :ref:`calendar ` CLIs -* :ref:`Binary releases for the experimental just-in-time compiler ` +* :ref:`Improved error messages ` +* :ref:`Incremental garbage collection ` +Significant improvements in the standard library: -Incompatible changes -==================== +* :pep:`784`: :ref:`Zstandard support in the standard library + ` +* :ref:`whatsnew314-asyncio-introspection` +* :ref:`whatsnew314-concurrent-warnings-control` +* :ref:`Syntax highlighting in the default interactive shell + `, and color output in several + standard library CLIs -On platforms other than macOS and Windows, the default :ref:`start -method ` for :mod:`multiprocessing` -and :class:`~concurrent.futures.ProcessPoolExecutor` switches from -*fork* to *forkserver*. +C API improvements: -See :ref:`(1) ` and -:ref:`(2) ` for details. +* :pep:`741`: :ref:`Python configuration C API ` -If you encounter :exc:`NameError`\s or pickling errors coming out of -:mod:`multiprocessing` or :mod:`concurrent.futures`, see the -:ref:`forkserver restrictions `. +Platform support: + +* :pep:`776`: Emscripten is now an :ref:`officially supported platform + `, at :pep:`tier 3 <11#tier-3>`. + +Release changes: + +* :pep:`779`: :ref:`Free-threaded Python is officially supported + ` +* :pep:`761`: :ref:`PGP signatures have been discontinued for official releases + ` +* :ref:`Windows and macOS binary releases now support the experimental + just-in-time compiler ` +* :ref:`Binary releases for Android are now provided ` -The interpreter avoids some reference count modifications internally when -it's safe to do so. This can lead to different values returned from -:func:`sys.getrefcount` and :c:func:`Py_REFCNT` compared to previous versions -of Python. See :ref:`below ` for details. New features ============ -.. _whatsnew314-pep779: +.. _whatsnew314-deferred-annotations: -PEP 779: Free-threaded Python is officially supported ------------------------------------------------------ - -The free-threaded build of Python is now supported and no longer experimental. -This is the start of phase II where free-threaded Python is officially supported -but still optional. - -We are confident that the project is on the right path, and we appreciate the -continued dedication from everyone working to make free-threading ready for -broader adoption across the Python community. - -With these recommendations and the acceptance of this PEP, we as the Python -developer community should broadly advertise that free-threading is a supported -Python build option now and into the future, and that it will not be removed -without a proper deprecation schedule. - -Any decision to transition to phase III, with free-threading as the default or -sole build of Python is still undecided, and dependent on many factors both -within CPython itself and the community. This decision is for the future. - -.. seealso:: - :pep:`779` and its `acceptance - `__. - -.. _whatsnew314-pep734: - -PEP 734: Multiple interpreters in the stdlib --------------------------------------------- - -The CPython runtime supports running multiple copies of Python in the -same process simultaneously and has done so for over 20 years. -Each of these separate copies is called an "interpreter". -However, the feature had been available only through the C-API. - -That limitation is removed in the 3.14 release, -with the new :mod:`concurrent.interpreters` module. - -There are at least two notable reasons why using multiple interpreters -is worth considering: - -* they support a new (to Python), human-friendly concurrency model -* true multi-core parallelism - -For some use cases, concurrency in software enables efficiency and -can simplify software, at a high level. At the same time, implementing -and maintaining all but the simplest concurrency is often a struggle -for the human brain. That especially applies to plain threads -(for example, :mod:`threading`), where all memory is shared between all threads. - -With multiple isolated interpreters, you can take advantage of a class -of concurrency models, like CSP or the actor model, that have found -success in other programming languages, like Smalltalk, Erlang, -Haskell, and Go. Think of multiple interpreters like threads -but with opt-in sharing. - -Regarding multi-core parallelism: as of the 3.12 release, interpreters -are now sufficiently isolated from one another to be used in parallel. -(See :pep:`684`.) This unlocks a variety of CPU-intensive use cases -for Python that were limited by the :term:`GIL`. - -Using multiple interpreters is similar in many ways to -:mod:`multiprocessing`, in that they both provide isolated logical -"processes" that can run in parallel, with no sharing by default. -However, when using multiple interpreters, an application will use -fewer system resources and will operate more efficiently (since it -stays within the same process). Think of multiple interpreters as -having the isolation of processes with the efficiency of threads. - -.. XXX Add an example or two. -.. XXX Link to the not-yet-added HOWTO doc. - -While the feature has been around for decades, multiple interpreters -have not been used widely, due to low awareness and the lack of a stdlib -module. Consequently, they currently have several notable limitations, -which will improve significantly now that the feature is finally -going mainstream. - -Current limitations: - -* starting each interpreter has not been optimized yet -* each interpreter uses more memory than necessary - (we will be working next on extensive internal sharing between - interpreters) -* there aren't many options *yet* for truly sharing objects or other - data between interpreters (other than :type:`memoryview`) -* many extension modules on PyPI are not compatible with multiple - interpreters yet (stdlib extension modules *are* compatible) -* the approach to writing applications that use multiple isolated - interpreters is mostly unfamiliar to Python users, for now - -The impact of these limitations will depend on future CPython -improvements, how interpreters are used, and what the community solves -through PyPI packages. Depending on the use case, the limitations may -not have much impact, so try it out! - -Furthermore, future CPython releases will reduce or eliminate overhead -and provide utilities that are less appropriate on PyPI. In the -meantime, most of the limitations can also be addressed through -extension modules, meaning PyPI packages can fill any gap for 3.14, and -even back to 3.12 where interpreters were finally properly isolated and -stopped sharing the :term:`GIL`. Likewise, we expect to slowly see -libraries on PyPI for high-level abstractions on top of interpreters. - -Regarding extension modules, work is in progress to update some PyPI -projects, as well as tools like Cython, pybind11, nanobind, and PyO3. -The steps for isolating an extension module are found at -:ref:`isolating-extensions-howto`. Isolating a module has a lot of -overlap with what is required to support -:ref:`free-threading `, -so the ongoing work in the community in that area will help accelerate -support for multiple interpreters. - -Also added in 3.14: :ref:`concurrent.futures.InterpreterPoolExecutor -`. - -.. seealso:: - :pep:`734`. - - -.. _whatsnew314-pep750: - -PEP 750: Template strings -------------------------- - -Template string literals (t-strings) are a generalization of f-strings, -using a ``t`` in place of the ``f`` prefix. Instead of evaluating -to :class:`str`, t-strings evaluate to a new :class:`!string.templatelib.Template` type: - -.. code-block:: python - - from string.templatelib import Template - - name = "World" - template: Template = t"Hello {name}" - -The template can then be combined with functions that operate on the template's -structure to produce a :class:`str` or a string-like result. -For example, sanitizing input: - -.. code-block:: python - - evil = "" - template = t"

{evil}

" - assert html(template) == "

<script>alert('evil')</script>

" - -As another example, generating HTML attributes from data: - -.. code-block:: python - - attributes = {"src": "shrubbery.jpg", "alt": "looks nice"} - template = t"" - assert html(template) == 'looks nice' - -Compared to using an f-string, the ``html`` function has access to template attributes -containing the original information: static strings, interpolations, and values -from the original scope. Unlike existing templating approaches, t-strings build -from the well-known f-string syntax and rules. Template systems thus benefit -from Python tooling as they are much closer to the Python language, syntax, -scoping, and more. - -Writing template handlers is straightforward: - -.. code-block:: python - - from string.templatelib import Template, Interpolation - - def lower_upper(template: Template) -> str: - """Render static parts lowercased and interpolations uppercased.""" - parts: list[str] = [] - for item in template: - if isinstance(item, Interpolation): - parts.append(str(item.value).upper()) - else: - parts.append(item.lower()) - return "".join(parts) - - name = "world" - assert lower_upper(t"HELLO {name}") == "hello WORLD" - -With this in place, developers can write template systems to sanitize SQL, make -safe shell operations, improve logging, tackle modern ideas in web development -(HTML, CSS, and so on), and implement lightweight, custom business DSLs. - -(Contributed by Jim Baker, Guido van Rossum, Paul Everitt, Koudai Aono, -Lysandros Nikolaou, Dave Peck, Adam Turner, Jelle Zijlstra, Bénédikt Tran, -and Pablo Galindo Salgado in :gh:`132661`.) - -.. seealso:: - :pep:`750`. - - -.. _whatsnew314-pep768: - -PEP 768: Safe external debugger interface for CPython ------------------------------------------------------ - -:pep:`768` introduces a zero-overhead debugging interface that allows debuggers and profilers -to safely attach to running Python processes. This is a significant enhancement to Python's -debugging capabilities allowing debuggers to forego unsafe alternatives. See -:ref:`below ` for how this feature is leveraged to -implement the new :mod:`pdb` module's remote attaching capabilities. - -The new interface provides safe execution points for attaching debugger code without modifying -the interpreter's normal execution path or adding runtime overhead. This enables tools to -inspect and interact with Python applications in real-time without stopping or restarting -them — a crucial capability for high-availability systems and production environments. - -For convenience, CPython implements this interface through the :mod:`sys` module with a -:func:`sys.remote_exec` function:: - - sys.remote_exec(pid, script_path) - -This function allows sending Python code to be executed in a target process at the next safe -execution point. However, tool authors can also implement the protocol directly as described -in the PEP, which details the underlying mechanisms used to safely attach to running processes. - -Here's a simple example that inspects object types in a running Python process: - - .. code-block:: python - - import os - import sys - import tempfile - - # Create a temporary script - with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: - script_path = f.name - f.write(f"import my_debugger; my_debugger.connect({os.getpid()})") - try: - # Execute in process with PID 1234 - print("Behold! An offering:") - sys.remote_exec(1234, script_path) - finally: - os.unlink(script_path) - -The debugging interface has been carefully designed with security in mind and includes several -mechanisms to control access: - -* A :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` environment variable. -* A :option:`-X disable-remote-debug` command-line option. -* A :option:`--without-remote-debug` configure flag to completely disable the feature at build time. - -A key implementation detail is that the interface piggybacks on the interpreter's existing evaluation -loop and safe points, ensuring zero overhead during normal execution while providing a reliable way -for external processes to coordinate debugging operations. - -(Contributed by Pablo Galindo Salgado, Matt Wozniski, and Ivona Stojanovic in :gh:`131591`.) - -.. seealso:: - :pep:`768`. - - -.. _whatsnew314-pep784: - -PEP 784: Adding Zstandard to the standard library -------------------------------------------------- - -The new ``compression`` package contains modules :mod:`!compression.lzma`, -:mod:`!compression.bz2`, :mod:`!compression.gzip` and :mod:`!compression.zlib` -which re-export the :mod:`lzma`, :mod:`bz2`, :mod:`gzip` and :mod:`zlib` -modules respectively. The new import names under ``compression`` are the -canonical names for importing these compression modules going forward. However, -the existing modules names have not been deprecated. Any deprecation or removal -of the existing compression modules will occur no sooner than five years after -the release of 3.14. - -The new :mod:`!compression.zstd` module provides compression and decompression -APIs for the Zstandard format via bindings to `Meta's zstd library -`__. Zstandard is a widely adopted, highly -efficient, and fast compression format. In addition to the APIs introduced in -:mod:`!compression.zstd`, support for reading and writing Zstandard compressed -archives has been added to the :mod:`tarfile`, :mod:`zipfile`, and -:mod:`shutil` modules. - -Here's an example of using the new module to compress some data: - -.. code-block:: python - - from compression import zstd - import math - - data = str(math.pi).encode() * 20 - - compressed = zstd.compress(data) - - ratio = len(compressed) / len(data) - print(f"Achieved compression ratio of {ratio}") - -As can be seen, the API is similar to the APIs of the :mod:`!lzma` and -:mod:`!bz2` modules. - -(Contributed by Emma Harper Smith, Adam Turner, Gregory P. Smith, Tomas Roun, -Victor Stinner, and Rogdham in :gh:`132983`.) - -.. seealso:: - :pep:`784`. - - -.. _whatsnew314-remote-pdb: - -Remote attaching to a running Python process with PDB ------------------------------------------------------ - -The :mod:`pdb` module now supports remote attaching to a running Python process -using a new ``-p PID`` command-line option: - -.. code-block:: sh - - python -m pdb -p 1234 - -This will connect to the Python process with the given PID and allow you to -debug it interactively. Notice that due to how the Python interpreter works -attaching to a remote process that is blocked in a system call or waiting for -I/O will only work once the next bytecode instruction is executed or when the -process receives a signal. - -This feature uses :pep:`768` and the :func:`sys.remote_exec` function -to attach to the remote process and send the PDB commands to it. - - -(Contributed by Matt Wozniski and Pablo Galindo in :gh:`131591`.) - -.. seealso:: - :pep:`768`. - - -.. _whatsnew314-pep758: - -PEP 758 – Allow except and except* expressions without parentheses ------------------------------------------------------------------- - -The :keyword:`except` and :keyword:`except* ` expressions now allow -parentheses to be omitted when there are multiple exception types and the ``as`` clause is not used. -For example the following expressions are now valid: - -.. code-block:: python - - try: - connect_to_server() - except TimeoutError, ConnectionRefusedError: - print("Network issue encountered.") - - # The same applies to except* (for exception groups): - - try: - connect_to_server() - except* TimeoutError, ConnectionRefusedError: - print("Network issue encountered.") - -Check :pep:`758` for more details. - -(Contributed by Pablo Galindo and Brett Cannon in :gh:`131831`.) - -.. seealso:: - :pep:`758`. - - -.. _whatsnew314-pep649: - -PEP 649 and 749: deferred evaluation of annotations ---------------------------------------------------- +:pep:`649` & :pep:`749`: Deferred evaluation of annotations +------------------------------------------------------------ The :term:`annotations ` on functions, classes, and modules are no longer evaluated eagerly. Instead, annotations are stored in special-purpose :term:`annotate functions ` and evaluated only when necessary (except if ``from __future__ import annotations`` is used). -This is specified in :pep:`649` and :pep:`749`. -This change is designed to make annotations in Python more performant and more -usable in most circumstances. The runtime cost for defining annotations is +This change is designed to improve performance and usability of annotations +in Python in most circumstances. The runtime cost for defining annotations is minimized, but it remains possible to introspect annotations at runtime. It is no longer necessary to enclose annotations in strings if they contain forward references. @@ -524,70 +182,344 @@ This example shows how these formats behave: >>> get_annotations(func, format=Format.STRING) {'arg': 'Undefined'} -Implications for annotated code -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The :ref:`porting ` section contains guidance +on changes that may be needed due to these changes, though in the majority of +cases, code will continue working as-is. -If you define annotations in your code (for example, for use with a static type -checker), then this change probably does not affect you: you can keep -writing annotations the same way you did with previous versions of Python. - -You will likely be able to remove quoted strings in annotations, which are frequently -used for forward references. Similarly, if you use ``from __future__ import annotations`` -to avoid having to write strings in annotations, you may well be able to -remove that import once you support only Python 3.14 and newer. -However, if you rely on third-party libraries that read annotations, -those libraries may need changes to support unquoted annotations before they -work as expected. - -Implications for readers of ``__annotations__`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If your code reads the ``__annotations__`` attribute on objects, you may want -to make changes in order to support code that relies on deferred evaluation of -annotations. For example, you may want to use :func:`annotationlib.get_annotations` -with the :attr:`~annotationlib.Format.FORWARDREF` format, as the :mod:`dataclasses` -module now does. - -The external :pypi:`typing_extensions` package provides partial backports of some of the -functionality of the :mod:`annotationlib` module, such as the :class:`~annotationlib.Format` -enum and the :func:`~annotationlib.get_annotations` function. These can be used to -write cross-version code that takes advantage of the new behavior in Python 3.14. - -Related changes -^^^^^^^^^^^^^^^ - -The changes in Python 3.14 are designed to rework how ``__annotations__`` -works at runtime while minimizing breakage to code that contains -annotations in source code and to code that reads ``__annotations__``. However, -if you rely on undocumented details of the annotation behavior or on private -functions in the standard library, there are many ways in which your code may -not work in Python 3.14. To safeguard your code against future changes, -use only the documented functionality of the :mod:`annotationlib` module. - -In particular, do not read annotations directly from the namespace dictionary -attribute of type objects. Use :func:`annotationlib.get_annotate_from_class_namespace` -during class construction and :func:`annotationlib.get_annotations` afterwards. - -In previous releases, it was sometimes possible to access class annotations from -an instance of an annotated class. This behavior was undocumented and accidental, -and will no longer work in Python 3.14. - -``from __future__ import annotations`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In Python 3.7, :pep:`563` introduced the ``from __future__ import annotations`` -directive, which turns all annotations into strings. This directive is now -considered deprecated and it is expected to be removed in a future version of Python. -However, this removal will not happen until after Python 3.13, the last version of -Python without deferred evaluation of annotations, reaches its end of life in 2029. -In Python 3.14, the behavior of code using ``from __future__ import annotations`` -is unchanged. - -(Contributed by Jelle Zijlstra in :gh:`119180`; :pep:`649` was written by Larry Hastings.) +(Contributed by Jelle Zijlstra in :pep:`749` and :gh:`119180`; +:pep:`649` was written by Larry Hastings.) .. seealso:: - :pep:`649` and :pep:`749`. + :pep:`649` + Deferred Evaluation Of Annotations Using Descriptors + :pep:`749` + Implementing PEP 649 + + +.. _whatsnew314-multiple-interpreters: + +:pep:`734`: Multiple interpreters in the standard library +--------------------------------------------------------- + +The CPython runtime supports running multiple copies of Python in the +same process simultaneously and has done so for over 20 years. +Each of these separate copies is called an 'interpreter'. +However, the feature had been available only through +the :ref:`C-API `. + +That limitation is removed in Python 3.14, +with the new :mod:`concurrent.interpreters` module. + +There are at least two notable reasons why using multiple interpreters +has significant benefits: + +* they support a new (to Python), human-friendly concurrency model +* true multi-core parallelism + +For some use cases, concurrency in software improves efficiency and +can simplify design, at a high level. +At the same time, implementing and maintaining all but the simplest concurrency +is often a struggle for the human brain. +That especially applies to plain threads (for example, :mod:`threading`), +where all memory is shared between all threads. + +With multiple isolated interpreters, you can take advantage of a class +of concurrency models, like Communicating Sequential Processes (CSP) +or the actor model, that have found +success in other programming languages, like Smalltalk, Erlang, +Haskell, and Go. Think of multiple interpreters as threads +but with opt-in sharing. + +Regarding multi-core parallelism: as of Python 3.12, interpreters +are now sufficiently isolated from one another to be used in parallel +(see :pep:`684`). This unlocks a variety of CPU-intensive use cases +for Python that were limited by the :term:`GIL`. + +Using multiple interpreters is similar in many ways to +:mod:`multiprocessing`, in that they both provide isolated logical +"processes" that can run in parallel, with no sharing by default. +However, when using multiple interpreters, an application will use +fewer system resources and will operate more efficiently (since it +stays within the same process). Think of multiple interpreters as +having the isolation of processes with the efficiency of threads. + +.. XXX Add an example or two. +.. XXX Link to the not-yet-added HOWTO doc. + +While the feature has been around for decades, multiple interpreters +have not been used widely, due to low awareness and the lack of a +standard library module. Consequently, they currently have several +notable limitations, which are expected to improve significantly now +that the feature is going mainstream. + +Current limitations: + +* starting each interpreter has not been optimized yet +* each interpreter uses more memory than necessary + (work continues on extensive internal sharing between interpreters) +* there aren't many options *yet* for truly sharing objects or other + data between interpreters (other than :type:`memoryview`) +* many third-party extension modules on PyPI are not yet compatible + with multiple interpreters + (all standard library extension modules *are* compatible) +* the approach to writing applications that use multiple isolated + interpreters is mostly unfamiliar to Python users, for now + +The impact of these limitations will depend on future CPython +improvements, how interpreters are used, and what the community solves +through PyPI packages. Depending on the use case, the limitations may +not have much impact, so try it out! + +Furthermore, future CPython releases will reduce or eliminate overhead +and provide utilities that are less appropriate on PyPI. In the +meantime, most of the limitations can also be addressed through +extension modules, meaning PyPI packages can fill any gap for 3.14, and +even back to 3.12 where interpreters were finally properly isolated and +stopped sharing the :term:`GIL`. Likewise, libraries on PyPI are expected +to emerge for high-level abstractions on top of interpreters. + +Regarding extension modules, work is in progress to update some PyPI +projects, as well as tools like Cython, pybind11, nanobind, and PyO3. +The steps for isolating an extension module are found at +:ref:`isolating-extensions-howto`. +Isolating a module has a lot of overlap with what is required to support +:ref:`free-threading `, so the ongoing +work in the community in that area will help accelerate support +for multiple interpreters. + +Also added in 3.14: :ref:`concurrent.futures.InterpreterPoolExecutor +`. + +(Contributed by Eric Snow in :gh:`134939`.) + +.. seealso:: :pep:`734` + + +.. _whatsnew314-template-string-literals: + +:pep:`750`: Template string literals +------------------------------------ + +Template strings are a new mechanism for custom string processing. +They share the familiar syntax of f-strings but, unlike f-strings, +return an object representing the static and interpolated parts of +the string, instead of a simple :class:`str`. + +To write a t-string, use a ``'t'`` prefix instead of an ``'f'``: + +.. doctest:: + + >>> variety = 'Stilton' + >>> template = t'Try some {variety} cheese!' + >>> type(template) + + +:class:`~string.templatelib.Template` objects provide access to the static +and interpolated (in curly braces) parts of a string *before* they are combined. +Iterate over :class:`!Template` instances to access their parts in order: + +.. testsetup:: + + variety = 'Stilton' + template = t'Try some {variety} cheese!' + +.. doctest:: + + >>> list(template) + ['Try some ', Interpolation('Stilton', 'variety', None, ''), ' cheese!'] + +It's easy to write (or call) code to process :class:`!Template` instances. +For example, here's a function that renders static parts lowercase and +:class:`~string.templatelib.Interpolation` instances uppercase: + +.. code-block:: python + + from string.templatelib import Interpolation + + def lower_upper(template): + """Render static parts lowercase and interpolations uppercase.""" + parts = [] + for part in template: + if isinstance(part, Interpolation): + parts.append(str(part.value).upper()) + else: + parts.append(part.lower()) + return ''.join(parts) + + name = 'Wenslydale' + template = t'Mister {name}' + assert lower_upper(template) == 'mister WENSLYDALE' + +Because :class:`!Template` instances distinguish between static strings and +interpolations at runtime, they can be useful for sanitising user input. +Writing a :func:`!html` function that escapes user input in HTML is an exercise +left to the reader! +Template processing code can provide improved flexibility. +For instance, a more advanced :func:`!html` function could accept +a :class:`!dict` of HTML attributes directly in the template: + +.. code-block:: python + + attributes = {'src': 'limburger.jpg', 'alt': 'lovely cheese'} + template = t'' + assert html(template) == 'lovely cheese' + +Of course, template processing code does not need to return a string-like result. +An even *more* advanced :func:`!html` could return a custom type representing +a DOM-like structure. + +With t-strings in place, developers can write systems that sanitise SQL, +make safe shell operations, improve logging, tackle modern ideas in web +development (HTML, CSS, and so on), and implement lightweight custom business DSLs. + +(Contributed by Jim Baker, Guido van Rossum, Paul Everitt, Koudai Aono, +Lysandros Nikolaou, Dave Peck, Adam Turner, Jelle Zijlstra, Bénédikt Tran, +and Pablo Galindo Salgado in :gh:`132661`.) + +.. seealso:: :pep:`750`. + + +.. _whatsnew314-remote-debugging: + +:pep:`768`: Safe external debugger interface +-------------------------------------------- + +Python 3.14 introduces a zero-overhead debugging interface that allows +debuggers and profilers to safely attach to running Python processes +without stopping or restarting them. +This is a significant enhancement to Python's debugging capabilities, +meaning that unsafe alternatives are no longer required. + +The new interface provides safe execution points for attaching debugger code +without modifying the interpreter's normal execution path +or adding any overhead at runtime. +Due to this, tools can now inspect and interact with Python applications +in real-time, which is a crucial capability for high-availability systems +and production environments. + +For convenience, this interface is implemented in the :func:`sys.remote_exec` +function. For example: + +.. code-block:: python + + import sys + from tempfile import NamedTemporaryFile + + with NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: + script_path = f.name + f.write(f'import my_debugger; my_debugger.connect({os.getpid()})') + + # Execute in process with PID 1234 + print('Behold! An offering:') + sys.remote_exec(1234, script_path) + + +This function allows sending Python code to be executed in a target process +at the next safe execution point. +However, tool authors can also implement the protocol directly as described +in the PEP, which details the underlying mechanisms used to safely attach to +running processes. + +The debugging interface has been carefully designed with security in mind +and includes several mechanisms to control access: + +* A :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` environment variable. +* A :option:`-X disable-remote-debug` command-line option. +* A :option:`--without-remote-debug` configure flag to completely disable + the feature at build time. + +(Contributed by Pablo Galindo Salgado, Matt Wozniski, and Ivona Stojanovic +in :gh:`131591`.) + +.. seealso:: :pep:`768`. + + +.. _whatsnew314-tail-call-interpreter: + +A new type of interpreter +------------------------- + +A new type of interpreter has been added to CPython. +It uses tail calls between small C functions that implement individual +Python opcodes, rather than one large C ``case`` statement. +For certain newer compilers, this interpreter provides +significantly better performance. Preliminary benchmarks suggest a geometric +mean of 3-5% faster on the standard ``pyperformance`` benchmark suite, +depending on platform and architecture. +The baseline is Python 3.14 built with Clang 19, without this new interpreter. + +This interpreter currently only works with Clang 19 and newer +on x86-64 and AArch64 architectures. +However, a future release of GCC is expected to support this as well. + +This feature is opt-in for now. Enabling profile-guided optimization is highly +recommended when using the new interpreter as it is the only configuration +that has been tested and validated for improved performance. +For further information, see :option:`--with-tail-call-interp`. + +.. note:: + + This is not to be confused with `tail call optimization`__ of Python + functions, which is currently not implemented in CPython. + + This new interpreter type is an internal implementation detail of the CPython + interpreter. It doesn't change the visible behavior of Python programs at + all. It can improve their performance, but doesn't change anything else. + + __ https://en.wikipedia.org/wiki/Tail_call + +(Contributed by Ken Jin in :gh:`128563`, with ideas on how to implement this +in CPython by Mark Shannon, Garrett Gu, Haoran Xu, and Josh Haberman.) + + +.. _whatsnew314-free-threaded-cpython: + +Free-threaded mode improvements +------------------------------- + +CPython's free-threaded mode (:pep:`703`), initially added in 3.13, +has been significantly improved in Python 3.14. +The implementation described in PEP 703 has been finished, including C API +changes, and temporary workarounds in the interpreter were replaced with +more permanent solutions. +The specializing adaptive interpreter (:pep:`659`) is now enabled +in free-threaded mode, which along with many other optimizations +greatly improves its performance. +The performance penalty on single-threaded code in free-threaded mode +is now roughly 5-10%, depending on the platform and C compiler used. + +From Python 3.14, when compiling extension modules for the free-threaded build of +CPython on Windows, the preprocessor variable ``Py_GIL_DISABLED`` now needs to +be specified by the build backend, as it will no longer be determined +automatically by the C compiler. For a running interpreter, the setting that +was used at compile time can be found using :func:`sysconfig.get_config_var`. + +The new :option:`-X context_aware_warnings <-X>` flag controls if +:ref:`concurrent safe warnings control ` +is enabled. The flag defaults to true for the free-threaded build +and false for the GIL-enabled build. + +A new :data:`~sys.flags.thread_inherit_context` flag has been added, +which if enabled means that threads created with :class:`threading.Thread` +start with a copy of the :class:`~contextvars.Context()` of the caller of +:meth:`~threading.Thread.start`. Most significantly, this makes the warning +filtering context established by :class:`~warnings.catch_warnings` be +"inherited" by threads (or asyncio tasks) started within that context. It also +affects other modules that use context variables, such as the :mod:`decimal` +context manager. +This flag defaults to true for the free-threaded build and false for +the GIL-enabled build. + +(Contributed by Sam Gross, Matt Page, Neil Schemenauer, Thomas Wouters, +Donghee Na, Kirill Podoprigora, Ken Jin, Itamar Oren, Brett Simmers, +Dino Viehland, Nathan Goldbaum, Ralf Gommers, Lysandros Nikolaou, Kumar Aditya, +Edgar Margffoy, and many others. +Some of these contributors are employed by Meta, which has continued to provide +significant engineering resources to support this project.) + + +.. _whatsnew314-improved-error-messages: Improved error messages ----------------------- @@ -608,47 +540,12 @@ Improved error messages ^^^^^^ SyntaxError: invalid syntax. Did you mean 'while'? - >>> asynch def fetch_data(): - ... pass - Traceback (most recent call last): - File "", line 1 - asynch def fetch_data(): - ^^^^^^ - SyntaxError: invalid syntax. Did you mean 'async'? - - >>> async def foo(): - ... awaid fetch_data() - Traceback (most recent call last): - File "", line 2 - awaid fetch_data() - ^^^^^ - SyntaxError: invalid syntax. Did you mean 'await'? - - >>> raisee ValueError("Error") - Traceback (most recent call last): - File "", line 1 - raisee ValueError("Error") - ^^^^^^ - SyntaxError: invalid syntax. Did you mean 'raise'? - While the feature focuses on the most common cases, some variations of misspellings may still result in regular syntax errors. (Contributed by Pablo Galindo in :gh:`132449`.) -* When an unpacking assignment fails due to an incorrect number of variables, the - error message prints the received number of values in more cases than before. - (Contributed by Tushar Sadhwani in :gh:`122239`.) - - .. code-block:: pycon - - >>> x, y, z = 1, 2, 3, 4 - Traceback (most recent call last): - File "", line 1, in - x, y, z = 1, 2, 3, 4 - ^^^^^^^ - ValueError: too many values to unpack (expected 3, got 4) - -* :keyword:`elif` statements that follow an :keyword:`else` block now have a specific error message. +* :keyword:`elif` statements that follow an :keyword:`else` block now have + a specific error message. (Contributed by Steele Farnsworth in :gh:`129902`.) .. code-block:: pycon @@ -664,11 +561,9 @@ Improved error messages ^^^^ SyntaxError: 'elif' block follows an 'else' block -* If a statement (:keyword:`pass`, :keyword:`del`, :keyword:`return`, - :keyword:`yield`, :keyword:`raise`, :keyword:`break`, :keyword:`continue`, - :keyword:`assert`, :keyword:`import`, :keyword:`from`) is passed to the - :ref:`if_expr` after :keyword:`else`, or one of :keyword:`pass`, - :keyword:`break`, or :keyword:`continue` is passed before :keyword:`if`, then the +* If a statement is passed to the :ref:`if_expr` after :keyword:`else`, + or one of :keyword:`pass`, :keyword:`break`, or :keyword:`continue` + is passed before :keyword:`if`, then the error message highlights where the :token:`~python-grammar:expression` is required. (Contributed by Sergey Miryanov in :gh:`129515`.) @@ -688,10 +583,9 @@ Improved error messages ^^^^^^^^ SyntaxError: expected expression before 'if', but statement is given - * When incorrectly closed strings are detected, the error message suggests - that the string may be intended to be part of the string. (Contributed by - Pablo Galindo in :gh:`88535`.) + that the string may be intended to be part of the string. + (Contributed by Pablo Galindo in :gh:`88535`.) .. code-block:: pycon @@ -700,8 +594,8 @@ Improved error messages SyntaxError: invalid syntax. Is this intended to be part of the string? * When strings have incompatible prefixes, the error now shows - which prefixes are incompatible. (Contributed by - Nikita Sobolev in :gh:`133197`.) + which prefixes are incompatible. + (Contributed by Nikita Sobolev in :gh:`133197`.) .. code-block:: pycon @@ -718,20 +612,11 @@ Improved error messages - Except handlers: ``except ... as ...`` - Pattern-match cases: ``case ... as ...`` - (Contributed by Nikita Sobolev in :gh:`123539`, - :gh:`123562`, and :gh:`123440`.) - - .. code-block:: pycon - - >>> import ast as arr[0] - File "", line 1 - import ast as arr[0] - ^^^^^^ - SyntaxError: cannot use subscript as import target + (Contributed by Nikita Sobolev in :gh:`123539`, :gh:`123562`, and :gh:`123440`.) * Improved error message when trying to add an instance of an unhashable type to - a :class:`dict` or :class:`set`. (Contributed by CF Bolz-Tereick and Victor Stinner - in :gh:`132828`.) + a :class:`dict` or :class:`set`. + (Contributed by CF Bolz-Tereick and Victor Stinner in :gh:`132828`.) .. code-block:: pycon @@ -751,66 +636,77 @@ Improved error messages ~^^^ TypeError: cannot use 'list' as a dict key (unhashable type: 'list') +* Improved error message when an object supporting the synchronous + context manager protocol is entered using :keyword:`async with` + instead of :keyword:`with`, + and vice versa for the asynchronous context manager protocol. + (Contributed by Bénédikt Tran in :gh:`128398`.) -.. _whatsnew314-pep741: -PEP 741: Python configuration C API ------------------------------------ +.. _whatsnew314-zstandard: -Add a :ref:`PyInitConfig C API ` to configure the Python -initialization without relying on C structures and the ability to make -ABI-compatible changes in the future. +:pep:`784`: Zstandard support in the standard library +----------------------------------------------------- -Complete the :pep:`587` :ref:`PyConfig C API ` by adding -:c:func:`PyInitConfig_AddModule` which can be used to add a built-in extension -module; a feature previously referred to as the “inittab”. +The new :mod:`!compression` package contains modules :mod:`!compression.lzma`, +:mod:`!compression.bz2`, :mod:`!compression.gzip` and :mod:`!compression.zlib` +which re-export the :mod:`lzma`, :mod:`bz2`, :mod:`gzip` and :mod:`zlib` +modules respectively. The new import names under :mod:`!compression` are the +preferred names for importing these compression modules from Python 3.14. However, +the existing modules names have not been deprecated. Any deprecation or removal +of the existing compression modules will occur no sooner than five years after +the release of 3.14. -Add :c:func:`PyConfig_Get` and :c:func:`PyConfig_Set` functions to get and set -the current runtime configuration. +The new :mod:`!compression.zstd` module provides compression and decompression +APIs for the Zstandard format via bindings to `Meta's zstd library +`__. Zstandard is a widely adopted, highly +efficient, and fast compression format. In addition to the APIs introduced in +:mod:`!compression.zstd`, support for reading and writing Zstandard compressed +archives has been added to the :mod:`tarfile`, :mod:`zipfile`, and +:mod:`shutil` modules. -PEP 587 “Python Initialization Configuration” unified all the ways to configure -the Python initialization. This PEP unifies also the configuration of the -Python preinitialization and the Python initialization in a single API. -Moreover, this PEP only provides a single choice to embed Python, instead of -having two “Python” and “Isolated” choices (PEP 587), to simplify the API -further. +Here's an example of using the new module to compress some data: -The lower level PEP 587 PyConfig API remains available for use cases with an -intentionally higher level of coupling to CPython implementation details (such -as emulating the full functionality of CPython’s CLI, including its -configuration mechanisms). +.. code-block:: python -(Contributed by Victor Stinner in :gh:`107954`.) + from compression import zstd + import math + + data = str(math.pi).encode() * 20 + compressed = zstd.compress(data) + ratio = len(compressed) / len(data) + print(f"Achieved compression ratio of {ratio}") + +As can be seen, the API is similar to the APIs of the :mod:`!lzma` and +:mod:`!bz2` modules. + +(Contributed by Emma Harper Smith, Adam Turner, Gregory P. Smith, Tomas Roun, +Victor Stinner, and Rogdham in :gh:`132983`.) + +.. seealso:: :pep:`784`. -.. seealso:: - :pep:`741`. .. _whatsnew314-asyncio-introspection: Asyncio introspection capabilities ---------------------------------- -Added a new command-line interface to inspect running Python processes using -asynchronous tasks, available via: +Added a new command-line interface to inspect running Python processes +using asynchronous tasks, available via ``python -m asyncio ps PID`` +or ``python -m asyncio pstree PID``. -.. code-block:: bash +The ``ps`` subcommand inspects the given process ID (PID) and displays +information about currently running asyncio tasks. +It outputs a task table: a flat listing of all tasks, their names, +their coroutine stacks, and which tasks are awaiting them. - python -m asyncio ps PID -This tool inspects the given process ID (PID) and displays information about -currently running asyncio tasks. It outputs a task table: a flat -listing of all tasks, their names, their coroutine stacks, and which tasks are -awaiting them. - -.. code-block:: bash - - python -m asyncio pstree PID - -This tool fetches the same information, but renders a visual async call tree, -showing coroutine relationships in a hierarchical format. This command is -particularly useful for debugging long-running or stuck asynchronous programs. -It can help developers quickly identify where a program is blocked, what tasks -are pending, and how coroutines are chained together. +The ``pstree`` subcommand fetches the same information, but instead renders a +visual async call tree, showing coroutine relationships in a hierarchical format. +This command is particularly useful for debugging long-running or stuck +asynchronous programs. +It can help developers quickly identify where a program is blocked, +what tasks are pending, and how coroutines are chained together. For example given this code: @@ -818,23 +714,25 @@ For example given this code: import asyncio - async def play(track): + async def play_track(track): await asyncio.sleep(5) - print(f"🎵 Finished: {track}") + print(f'🎵 Finished: {track}') - async def album(name, tracks): + async def play_album(name, tracks): async with asyncio.TaskGroup() as tg: for track in tracks: - tg.create_task(play(track), name=track) + tg.create_task(play_track(track), name=track) async def main(): async with asyncio.TaskGroup() as tg: tg.create_task( - album("Sundowning", ["TNDNBTG", "Levitate"]), name="Sundowning") + play_album('Sundowning', ['TNDNBTG', 'Levitate']), + name='Sundowning') tg.create_task( - album("TMBTE", ["DYWTYLM", "Aqua Regia"]), name="TMBTE") + play_album('TMBTE', ['DYWTYLM', 'Aqua Regia']), + name='TMBTE') - if __name__ == "__main__": + if __name__ == '__main__': asyncio.run(main()) Executing the new tool on the running process will yield a table like this: @@ -899,169 +797,180 @@ prevent tree construction: (Contributed by Pablo Galindo, Łukasz Langa, Yury Selivanov, and Marta Gomez Macias in :gh:`91048`.) -.. _whatsnew314-tail-call: -A new type of interpreter -------------------------- - -A new type of interpreter has been added to CPython. -It uses tail calls between small C functions that implement individual -Python opcodes, rather than one large C case statement. -For certain newer compilers, this interpreter provides -significantly better performance. Preliminary numbers on our machines suggest -anywhere up to 30% faster Python code, and a geometric mean of 3-5% -faster on ``pyperformance`` depending on platform and architecture. The -baseline is Python 3.14 built with Clang 19 without this new interpreter. - -This interpreter currently only works with Clang 19 and newer -on x86-64 and AArch64 architectures. However, we expect -that a future release of GCC will support this as well. - -This feature is opt-in for now. We highly recommend enabling profile-guided -optimization with the new interpreter as it is the only configuration we have -tested and can validate its improved performance. -For further information on how to build Python, see -:option:`--with-tail-call-interp`. - -.. note:: - - This is not to be confused with `tail call optimization`__ of Python - functions, which is currently not implemented in CPython. - - This new interpreter type is an internal implementation detail of the CPython - interpreter. It doesn't change the visible behavior of Python programs at - all. It can improve their performance, but doesn't change anything else. - - __ https://en.wikipedia.org/wiki/Tail_call - -.. attention:: - - This section previously reported a 9-15% geometric mean speedup. This number has since been - cautiously revised down to 3-5%. While we expect performance results to be better - than what we report, our estimates are more conservative due to a - `compiler bug `_ found in - Clang/LLVM 19, which causes the normal interpreter to be slower. We were unaware of this bug, - resulting in inaccurate results. We sincerely apologize for - communicating results that were only accurate for LLVM v19.1.x and v20.1.0. In the meantime, - the bug has been fixed in LLVM v20.1.1 and for the upcoming v21.1, but it will remain - unfixed for LLVM v19.1.x and v20.1.0. Thus - any benchmarks with those versions of LLVM may produce inaccurate numbers. - (Thanks to Nelson Elhage for bringing this to light.) - -(Contributed by Ken Jin in :gh:`128563`, with ideas on how to implement this -in CPython by Mark Shannon, Garrett Gu, Haoran Xu, and Josh Haberman.) - -.. _whatsnew314-free-threaded-cpython: - -Free-threaded mode ------------------- - -Free-threaded mode (:pep:`703`), initially added in 3.13, has been significantly improved. -The implementation described in PEP 703 was finished, including C API changes, -and temporary workarounds in the interpreter were replaced with more permanent solutions. -The specializing adaptive interpreter (:pep:`659`) is now enabled in free-threaded mode, -which along with many other optimizations greatly improves its performance. -The performance penalty on single-threaded code in free-threaded mode is now roughly 5-10%, -depending on platform and C compiler used. - -This work was done by many contributors: Sam Gross, Matt Page, Neil Schemenauer, -Thomas Wouters, Donghee Na, Kirill Podoprigora, Ken Jin, Itamar Oren, -Brett Simmers, Dino Viehland, Nathan Goldbaum, Ralf Gommers, Lysandros Nikolaou, -Kumar Aditya, Edgar Margffoy, and many others. - -Some of these contributors are employed by Meta, which has continued to provide -significant engineering resources to support this project. - -From 3.14, when compiling extension modules for the free-threaded build of -CPython on Windows, the preprocessor variable ``Py_GIL_DISABLED`` now needs to -be specified by the build backend, as it will no longer be determined -automatically by the C compiler. For a running interpreter, the setting that -was used at compile time can be found using :func:`sysconfig.get_config_var`. - -A new flag has been added, :data:`~sys.flags.context_aware_warnings`. This -flag defaults to true for the free-threaded build and false for the GIL-enabled -build. If the flag is true then the :class:`warnings.catch_warnings` context -manager uses a context variable for warning filters. This makes the context -manager behave predictably when used with multiple threads or asynchronous -tasks. - -A new flag has been added, :data:`~sys.flags.thread_inherit_context`. This flag -defaults to true for the free-threaded build and false for the GIL-enabled -build. If the flag is true then threads created with :class:`threading.Thread` -start with a copy of the :class:`~contextvars.Context()` of the caller of -:meth:`~threading.Thread.start`. Most significantly, this makes the warning -filtering context established by :class:`~warnings.catch_warnings` be -"inherited" by threads (or asyncio tasks) started within that context. It also -affects other modules that use context variables, such as the :mod:`decimal` -context manager. - - -.. _whatsnew314-pyrepl-highlighting: - -Syntax highlighting in PyREPL ------------------------------ - -The default :term:`interactive` shell now highlights Python syntax as you -type. The feature is enabled by default unless the -:envvar:`PYTHON_BASIC_REPL` environment is set or any color-disabling -environment variables are used. See :ref:`using-on-controlling-color` for -details. - -The default color theme for syntax highlighting strives for good contrast -and uses exclusively the 4-bit VGA standard ANSI color codes for maximum -compatibility. The theme can be customized using an experimental API -``_colorize.set_theme()``. This can be called interactively, as well as -in the :envvar:`PYTHONSTARTUP` script. - -(Contributed by Łukasz Langa in :gh:`131507`.) - - -.. _whatsnew314-jit-compiler: - -Binary releases for the experimental just-in-time compiler ----------------------------------------------------------- - -The official macOS and Windows release binaries now include an *experimental* -just-in-time (JIT) compiler. Although it is **not** recommended for production -use, it can be tested by setting :envvar:`PYTHON_JIT=1 ` as an -environment variable. Downstream source builds and redistributors can use the -:option:`--enable-experimental-jit=yes-off` configuration option for similar -behavior. - -The JIT is at an early stage and still in active development. As such, the -typical performance impact of enabling it can range from 10% slower to 20% -faster, depending on workload. To aid in testing and evaluation, a set of -introspection functions has been provided in the :data:`sys._jit` namespace. -:func:`sys._jit.is_available` can be used to determine if the current executable -supports JIT compilation, while :func:`sys._jit.is_enabled` can be used to tell -if JIT compilation has been enabled for the current process. - -Currently, the most significant missing functionality is that native debuggers -and profilers like ``gdb`` and ``perf`` are unable to unwind through JIT frames -(Python debuggers and profilers, like :mod:`pdb` or :mod:`profile`, continue to -work without modification). Free-threaded builds do not support JIT compilation. - -Please report any bugs or major performance regressions that you encounter! - -.. seealso:: :pep:`744` +.. _whatsnew314-concurrent-warnings-control: Concurrent safe warnings control -------------------------------- The :class:`warnings.catch_warnings` context manager will now optionally -use a context variable for warning filters. This is enabled by setting +use a context variable for warning filters. This is enabled by setting the :data:`~sys.flags.context_aware_warnings` flag, either with the ``-X`` -command-line option or an environment variable. This gives predictable +command-line option or an environment variable. This gives predictable warnings control when using :class:`~warnings.catch_warnings` combined with multiple threads or asynchronous tasks. The flag defaults to true for the free-threaded build and false for the GIL-enabled build. (Contributed by Neil Schemenauer and Kumar Aditya in :gh:`130010`.) + +Other language changes +====================== + +* All Windows code pages are now supported as 'cpXXX' codecs on Windows. + (Contributed by Serhiy Storchaka in :gh:`123803`.) + +* Implement mixed-mode arithmetic rules combining real and complex numbers + as specified by the C standard since C99. + (Contributed by Sergey B Kirpichev in :gh:`69639`.) + +* More syntax errors are now detected regardless of optimisation and + the :option:`-O` command-line option. + This includes writes to ``__debug__``, incorrect use of :keyword:`await`, + and asynchronous comprehensions outside asynchronous functions. + For example, ``python -O -c 'assert (__debug__ := 1)'`` + or ``python -O -c 'assert await 1'`` now produce :exc:`SyntaxError`\ s. + (Contributed by Irit Katriel and Jelle Zijlstra in :gh:`122245` & :gh:`121637`.) + +* When subclassing a pure C type, the C slots for the new type + are no longer replaced with a wrapped version on class creation + if they are not explicitly overridden in the subclass. + (Contributed by Tomasz Pytel in :gh:`132284`.) + + +Built-ins +--------- + +* The :meth:`bytes.fromhex` and :meth:`bytearray.fromhex` methods now accept + ASCII :class:`bytes` and :term:`bytes-like objects `. + (Contributed by Daniel Pope in :gh:`129349`.) + +* Add class methods :meth:`float.from_number` and :meth:`complex.from_number` + to convert a number to :class:`float` or :class:`complex` type correspondingly. + They raise a :exc:`TypeError` if the argument is not a real number. + (Contributed by Serhiy Storchaka in :gh:`84978`.) + +* Support underscore and comma as thousands separators in the fractional part + for floating-point presentation types of the new-style string formatting + (with :func:`format` or :ref:`f-strings`). + (Contributed by Sergey B Kirpichev in :gh:`87790`.) + +* The :func:`int` function no longer delegates to :meth:`~object.__trunc__`. + Classes that want to support conversion to :func:`!int` must implement + either :meth:`~object.__int__` or :meth:`~object.__index__`. + (Contributed by Mark Dickinson in :gh:`119743`.) + +* The :func:`map` function now has an optional keyword-only *strict* flag + like :func:`zip` to check that all the iterables are of equal length. + (Contributed by Wannes Boeykens in :gh:`119793`.) + +* The :class:`memoryview` type now supports subscription, + making it a :term:`generic type`. + (Contributed by Brian Schubert in :gh:`126012`.) + +* Using :data:`NotImplemented` in a boolean context + will now raise a :exc:`TypeError`. + This has raised a :exc:`DeprecationWarning` since Python 3.9. + (Contributed by Jelle Zijlstra in :gh:`118767`.) + +* Three-argument :func:`pow` now tries calling :meth:`~object.__rpow__` + if necessary. + Previously it was only called in two-argument :func:`!pow` + and the binary power operator. + (Contributed by Serhiy Storchaka in :gh:`130104`.) + +* :class:`super` objects are now :mod:`copyable ` and :mod:`pickleable + `. + (Contributed by Serhiy Storchaka in :gh:`125767`.) + + +Command line and environment +---------------------------- + +* The import time flag can now track modules that are already loaded ('cached'), + via the new :option:`-X importtime=2 <-X>`. + When such a module is imported, the ``self`` and ``cumulative`` times + are replaced by the string ``cached``. + + Values above ``2`` for ``-X importtime`` are now reserved for future use. + + (Contributed by Noah Kim and Adam Turner in :gh:`118655`.) + +* The command-line option :option:`-c` now automatically dedents its code + argument before execution. + (Contributed by Jon Crall and Steven Sun in :gh:`103998`.) + +* :option:`!-J` is no longer a reserved flag for Jython_, + and now has no special meaning. + (Contributed by Adam Turner in :gh:`133336`.) + + .. _Jython: https://www.jython.org/ + + +.. _whatsnew314-bracketless-except: + +PEP 758: Allow ``except`` and ``except*`` expressions without brackets +---------------------------------------------------------------------- + +The :keyword:`except` and :keyword:`except* ` expressions +now allow brackets to be omitted when there are multiple exception types +and the ``as`` clause is not used. +For example: + +.. code-block:: python + + try: + connect_to_server() + except TimeoutError, ConnectionRefusedError: + print('The network has ceased to be!') + +(Contributed by Pablo Galindo and Brett Cannon in :pep:`758` and :gh:`131831`.) + + +.. _whatsnew314-finally-syntaxwarning: + +PEP 765: Control flow in :keyword:`finally` blocks +-------------------------------------------------- + +The compiler now emits a :exc:`SyntaxWarning` when a :keyword:`return`, +:keyword:`break`, or :keyword:`continue` statement have the effect of +leaving a :keyword:`finally` block. +This change is specified in :pep:`765`. + +In situations where this change is inconvenient (such as those where the +warnings are redundant due to code linting), the :ref:`warning filter +` can be used to turn off all syntax warnings by adding +``ignore::SyntaxWarning`` as a filter. This can be specified in combination +with a filter that converts other warnings to errors (for example, passing +``-Werror -Wignore::SyntaxWarning`` as CLI options, or setting +``PYTHONWARNINGS=error,ignore::SyntaxWarning``). + +Note that applying such a filter at runtime using the :mod:`warnings` module +will only suppress the warning in code that is compiled *after* the filter is +adjusted. Code that is compiled prior to the filter adjustment (for example, +when a module is imported) will still emit the syntax warning. + +(Contributed by Irit Katriel in :gh:`130080`.) + + +.. _incremental-garbage-collection: .. _whatsnew314-incremental-gc: -Incremental garbage collection ------------------------------- +Garbage collection +------------------ + +**From Python 3.14.5 onwards:** + +The garbage collector (GC) has changed in Python 3.14.5. + +Python 3.14.0-3.14.4 shipped with a new incremental GC. +However, due to a number of `reports +`__ +of significant memory pressure in production environments, +it has been reverted back to the generational GC from 3.13. +This is the GC now used in Python 3.14.5 and later. + +**Previously in Python 3.14.0-3.14.4:** The cycle garbage collector is now incremental. This means that maximum pause times are reduced @@ -1081,157 +990,62 @@ The behavior of :func:`!gc.collect` changes slightly: (Contributed by Mark Shannon in :gh:`108362`.) -Platform support -================ -* :pep:`776`: Emscripten is now an officially supported platform at - :pep:`tier 3 <11#tier-3>`. As a part of this effort, more than 25 bugs in - `Emscripten libc`__ were fixed. Emscripten now includes support - for :mod:`ctypes`, :mod:`termios`, and :mod:`fcntl`, as well as - experimental support for :ref:`PyREPL `. +Default interactive shell +------------------------- - (Contributed by R. Hood Chatham in :gh:`127146`, :gh:`127683`, and :gh:`136931`.) +.. _whatsnew314-pyrepl-highlighting: - __ https://emscripten.org/docs/porting/emscripten-runtime-environment.html +* The default :term:`interactive` shell now highlights Python syntax. + The feature is enabled by default, save if :envvar:`PYTHON_BASIC_REPL` + or any other environment variable that disables colour is set. + See :ref:`using-on-controlling-color` for details. -Other language changes -====================== + The default color theme for syntax highlighting strives for good contrast + and exclusively uses the 4-bit VGA standard ANSI color codes for maximum + compatibility. The theme can be customized using an experimental API + :func:`!_colorize.set_theme`. + This can be called interactively or in the :envvar:`PYTHONSTARTUP` script. + Note that this function has no stability guarantees, + and may change or be removed. -* The default :term:`interactive` shell now supports import autocompletion. - This means that typing ``import foo`` and pressing ```` will suggest - modules starting with ``foo``. Similarly, typing ``from foo import b`` will - suggest submodules of ``foo`` starting with ``b``. Note that autocompletion - of module attributes is not currently supported. + (Contributed by Łukasz Langa in :gh:`131507`.) + +* The default :term:`interactive` shell now supports import auto-completion. + This means that typing ``import co`` and pressing :kbd:`` will suggest + modules starting with ``co``. Similarly, typing ``from concurrent import i`` + will suggest submodules of ``concurrent`` starting with ``i``. + Note that autocompletion of module attributes is not currently supported. (Contributed by Tomas Roun in :gh:`69605`.) -* The :func:`map` built-in now has an optional keyword-only *strict* flag - like :func:`zip` to check that all the iterables are of equal length. - (Contributed by Wannes Boeykens in :gh:`119793`.) - -* Incorrect usage of :keyword:`await` and asynchronous comprehensions - is now detected even if the code is optimized away by the :option:`-O` - command-line option. For example, ``python -O -c 'assert await 1'`` - now produces a :exc:`SyntaxError`. (Contributed by Jelle Zijlstra in :gh:`121637`.) - -* Writes to ``__debug__`` are now detected even if the code is optimized - away by the :option:`-O` command-line option. For example, - ``python -O -c 'assert (__debug__ := 1)'`` now produces a - :exc:`SyntaxError`. (Contributed by Irit Katriel in :gh:`122245`.) - -* Add class methods :meth:`float.from_number` and :meth:`complex.from_number` - to convert a number to :class:`float` or :class:`complex` type correspondingly. - They raise an error if the argument is a string. - (Contributed by Serhiy Storchaka in :gh:`84978`.) - -* Implement mixed-mode arithmetic rules combining real and complex numbers as - specified by C standards since C99. - (Contributed by Sergey B Kirpichev in :gh:`69639`.) - -* All Windows code pages are now supported as "cpXXX" codecs on Windows. - (Contributed by Serhiy Storchaka in :gh:`123803`.) - -* :class:`super` objects are now :mod:`pickleable ` and - :mod:`copyable `. - (Contributed by Serhiy Storchaka in :gh:`125767`.) - -* The :class:`memoryview` type now supports subscription, - making it a :term:`generic type`. - (Contributed by Brian Schubert in :gh:`126012`.) - -* Support underscore and comma as thousands separators in the fractional part - for floating-point presentation types of the new-style string formatting - (with :func:`format` or :ref:`f-strings`). - (Contributed by Sergey B Kirpichev in :gh:`87790`.) - -* The :func:`bytes.fromhex` and :func:`bytearray.fromhex` methods now accept - ASCII :class:`bytes` and :term:`bytes-like objects `. - (Contributed by Daniel Pope in :gh:`129349`.) - -* Support ``\z`` as a synonym for ``\Z`` in :mod:`regular expressions `. - It is interpreted unambiguously in many other regular expression engines, - unlike ``\Z``, which has subtly different behavior. - (Contributed by Serhiy Storchaka in :gh:`133306`.) - -* ``\B`` in :mod:`regular expression ` now matches the empty input string. - Now it is always the opposite of ``\b``. - (Contributed by Serhiy Storchaka in :gh:`124130`.) - -* iOS and macOS apps can now be configured to redirect ``stdout`` and - ``stderr`` content to the system log. (Contributed by Russell Keith-Magee in - :gh:`127592`.) - -* The iOS testbed is now able to stream test output while the test is running. - The testbed can also be used to run the test suite of projects other than - CPython itself. (Contributed by Russell Keith-Magee in :gh:`127592`.) - -* Three-argument :func:`pow` now tries calling :meth:`~object.__rpow__` if - necessary. Previously it was only called in two-argument :func:`!pow` and the - binary power operator. - (Contributed by Serhiy Storchaka in :gh:`130104`.) - -* Add a built-in implementation for HMAC (:rfc:`2104`) using formally verified - code from the `HACL* `__ project. - This implementation is used as a fallback when the OpenSSL implementation - of HMAC is not available. - (Contributed by Bénédikt Tran in :gh:`99108`.) - -* The import time flag can now track modules that are already loaded ('cached'), - via the new :option:`-X importtime=2 <-X>`. - When such a module is imported, the ``self`` and ``cumulative`` times - are replaced by the string ``cached``. - Values above ``2`` for ``-X importtime`` are now reserved for future use. - (Contributed by Noah Kim and Adam Turner in :gh:`118655`.) - -* When subclassing from a pure C type, the C slots for the new type are no - longer replaced with a wrapped version on class creation if they are not - explicitly overridden in the subclass. - (Contributed by Tomasz Pytel in :gh:`132329`.) - -* The command-line option :option:`-c` now automatically dedents its code - argument before execution. The auto-dedentation behavior mirrors - :func:`textwrap.dedent`. - (Contributed by Jon Crall and Steven Sun in :gh:`103998`.) - -* Improve error message when an object supporting the synchronous - context manager protocol is entered using :keyword:`async - with` instead of :keyword:`with`. - And vice versa with the asynchronous context manager protocol. - (Contributed by Bénédikt Tran in :gh:`128398`.) - -* :option:`!-J` is no longer a reserved flag for Jython_, - and now has no special meaning. - (Contributed by Adam Turner in :gh:`133336`.) - - .. _Jython: https://www.jython.org/ - -* The :func:`int` built-in no longer delegates to :meth:`~object.__trunc__`. - Classes that want to support conversion to :func:`!int` must implement - either :meth:`~object.__int__` or :meth:`~object.__index__`. - (Contributed by Mark Dickinson in :gh:`119743`.) - -* Using :data:`NotImplemented` in a boolean context - will now raise a :exc:`TypeError`. - This has raised a :exc:`DeprecationWarning` since Python 3.9. - (Contributed by Jelle Zijlstra in :gh:`118767`.) - - -.. _whatsnew314-pep765: - -PEP 765: Disallow ``return``/``break``/``continue`` that exit a ``finally`` block ---------------------------------------------------------------------------------- - -The compiler emits a :exc:`SyntaxWarning` when a :keyword:`return`, :keyword:`break` or -:keyword:`continue` statement appears where it exits a :keyword:`finally` block. -This change is specified in :pep:`765`. - New modules =========== -* :mod:`annotationlib`: For introspecting :term:`annotations `. - See :pep:`749` for more details. +* :mod:`annotationlib`: + For introspecting :term:`annotations `. + See :ref:`PEP 749 ` for more details. (Contributed by Jelle Zijlstra in :gh:`119180`.) +* :mod:`compression` (including :mod:`compression.zstd`): + A package for compression-related modules, + including a new module to support the Zstandard compression format. + See :ref:`PEP 784 ` for more details. + (Contributed by Emma Harper Smith, Adam Turner, Gregory P. Smith, Tomas Roun, + Victor Stinner, and Rogdham in :gh:`132983`.) + +* :mod:`concurrent.interpreters`: + Support for multiple interpreters in the standard library. + See :ref:`PEP 734 ` for more details. + (Contributed by Eric Snow in :gh:`134939`.) + +* :mod:`string.templatelib`: + Support for template string literals (t-strings). + See :ref:`PEP 750 ` for more details. + (Contributed by Jim Baker, Guido van Rossum, Paul Everitt, Koudai Aono, + Lysandros Nikolaou, Dave Peck, Adam Turner, Jelle Zijlstra, Bénédikt Tran, + and Pablo Galindo Salgado in :gh:`132661`.) + Improved modules ================ @@ -1249,8 +1063,6 @@ argparse and subparser names if mistyped by the user. (Contributed by Savannah Ostrowski in :gh:`124456`.) - .. _whatsnew314-color-argparse: - * Enable color for help text, which can be disabled with the optional *color* parameter to :class:`argparse.ArgumentParser`. This can also be controlled by :ref:`environment variables @@ -1261,7 +1073,7 @@ argparse ast --- -* Add :func:`ast.compare` for comparing two ASTs. +* Add :func:`~ast.compare`, a function for comparing two ASTs. (Contributed by Batuhan Taskaya and Jeremy Hylton in :gh:`60191`.) * Add support for :func:`copy.replace` for AST nodes. @@ -1270,15 +1082,17 @@ ast * Docstrings are now removed from an optimized AST in optimization level 2. (Contributed by Irit Katriel in :gh:`123958`.) -* The ``repr()`` output for AST nodes now includes more information. +* The :func:`repr` output for AST nodes now includes more information. (Contributed by Tomas Roun in :gh:`116022`.) -* :func:`ast.parse`, when called with an AST as input, now always verifies - that the root node type is appropriate. +* When called with an AST as input, the :func:`~ast.parse` function + now always verifies that the root node type is appropriate. (Contributed by Irit Katriel in :gh:`130139`.) -* Add new ``--feature-version``, ``--optimize``, ``--show-empty`` options to - the command-line interface. +* Add new options to the command-line interface: + :option:`--feature-version `, + :option:`--optimize `, and + :option:`--show-empty `. (Contributed by Semyon Moroz in :gh:`133367`.) @@ -1297,20 +1111,23 @@ asyncio :meth:`asyncio.create_task`, :meth:`asyncio.loop.create_task`, :meth:`asyncio.TaskGroup.create_task`. + (Contributed by Thomas Grainger in :gh:`128307`.) * There are two new utility functions for introspecting and printing a program's call graph: :func:`~asyncio.capture_call_graph` and :func:`~asyncio.print_call_graph`. + See :ref:`Asyncio introspection capabilities + ` for more details. (Contributed by Yury Selivanov, Pablo Galindo Salgado, and Łukasz Langa in :gh:`91048`.) - .. _whatsnew314-color-calendar: - calendar -------- +.. _whatsnew314-color-calendar: + * By default, today's date is highlighted in color in :mod:`calendar`'s :ref:`command-line ` text output. This can be controlled by :ref:`environment variables @@ -1323,19 +1140,25 @@ concurrent.futures .. _whatsnew314-concurrent-futures-interp-pool: -* Add :class:`~concurrent.futures.InterpreterPoolExecutor`, - which exposes "subinterpreters" (multiple Python interpreters in the - same process) to Python code. This is separate from the proposed API - in :pep:`734`. +* Add a new executor class, :class:`~concurrent.futures.InterpreterPoolExecutor`, + which exposes multiple Python interpreters in the same process + ('subinterpreters') to Python code. + This uses a pool of independent Python interpreters to execute calls + asynchronously. + + This is separate from the new :mod:`~concurrent.interpreters` module + introduced by :ref:`PEP 734 `. (Contributed by Eric Snow in :gh:`124548`.) .. _whatsnew314-concurrent-futures-start-method: -* The default :class:`~concurrent.futures.ProcessPoolExecutor` - :ref:`start method ` changed - from :ref:`fork ` to :ref:`forkserver - ` on platforms other than macOS and - Windows where it was already :ref:`spawn `. +* On Unix platforms other than macOS, :ref:`'forkserver' + ` is now the default :ref:`start + method ` for + :class:`~concurrent.futures.ProcessPoolExecutor` + (replacing :ref:`'fork' `). + This change does not affect Windows or macOS, where :ref:`'spawn' + ` remains the default start method. If the threading incompatible *fork* method is required, you must explicitly request it by supplying a multiprocessing context *mp_context* to @@ -1348,31 +1171,36 @@ concurrent.futures (Contributed by Gregory P. Smith in :gh:`84559`.) -* Add :meth:`concurrent.futures.ProcessPoolExecutor.terminate_workers` and - :meth:`concurrent.futures.ProcessPoolExecutor.kill_workers` as - ways to terminate or kill all living worker processes in the given pool. +* Add two new methods to :class:`~concurrent.futures.ProcessPoolExecutor`, + :meth:`~concurrent.futures.ProcessPoolExecutor.terminate_workers` + and :meth:`~concurrent.futures.ProcessPoolExecutor.kill_workers`, + as ways to terminate or kill all living worker processes in the given pool. (Contributed by Charles Machalow in :gh:`130849`.) -* Add the optional ``buffersize`` parameter to - :meth:`concurrent.futures.Executor.map` to limit the number of submitted +* Add the optional *buffersize* parameter to :meth:`Executor.map + ` to limit the number of submitted tasks whose results have not yet been yielded. If the buffer is full, iteration over the *iterables* pauses until a result is yielded from the buffer. (Contributed by Enzo Bonnal and Josh Rosenberg in :gh:`74028`.) + configparser ------------ -* Security fix: will no longer write config files it cannot read. Attempting - to :meth:`configparser.ConfigParser.write` keys containing delimiters or - beginning with the section header pattern will raise a - :class:`configparser.InvalidWriteError`. +* :mod:`!configparser` will no longer write config files it cannot read, + to improve security. + Attempting to :meth:`~configparser.ConfigParser.write` keys containing + delimiters or beginning with the section header pattern will raise an + :class:`~configparser.InvalidWriteError`. (Contributed by Jacob Lincoln in :gh:`129270`.) + contextvars ----------- -* Support context manager protocol by :class:`contextvars.Token`. +* Support the :term:`context manager` protocol + for :class:`~contextvars.Token` objects. (Contributed by Andrew Svetlov in :gh:`129889`.) @@ -1380,8 +1208,8 @@ ctypes ------ * The layout of :ref:`bit fields ` - in :class:`~ctypes.Structure` and :class:`~ctypes.Union` - now matches platform defaults (GCC/Clang or MSVC) more closely. + in :class:`~ctypes.Structure` and :class:`~ctypes.Union` objects + is now a closer match to platform defaults (GCC/Clang or MSVC). In particular, fields no longer overlap. (Contributed by Matthias Görgens in :gh:`97702`.) @@ -1400,7 +1228,7 @@ ctypes * On Windows, the :func:`~ctypes.CopyComPointer` function is now public. (Contributed by Jun Komoda in :gh:`127275`.) -* :func:`ctypes.memoryview_at` now exists to create a +* Add :func:`~ctypes.memoryview_at`, a function to create a :class:`memoryview` object that refers to the supplied pointer and length. This works like :func:`ctypes.string_at` except it avoids a buffer copy, and is typically useful when implementing pure Python @@ -1408,7 +1236,7 @@ ctypes (Contributed by Rian Hunter in :gh:`112018`.) * Complex types, :class:`~ctypes.c_float_complex`, - :class:`~ctypes.c_double_complex` and :class:`~ctypes.c_longdouble_complex`, + :class:`~ctypes.c_double_complex`, and :class:`~ctypes.c_longdouble_complex`, are now available if both the compiler and the ``libffi`` library support complex C types. (Contributed by Sergey B Kirpichev in :gh:`61103`.) @@ -1418,50 +1246,57 @@ ctypes (Contributed by Brian Ward in :gh:`119349`.) * Move :func:`ctypes.POINTER` types cache from a global internal cache - (``_pointer_type_cache``) to the :attr:`ctypes._CData.__pointer_type__` - attribute of the corresponding :mod:`ctypes` types. + (``_pointer_type_cache``) to the :attr:`_CData.__pointer_type__ + ` attribute of the corresponding + :mod:`!ctypes` types. This will stop the cache from growing without limits in some situations. (Contributed by Sergey Miryanov in :gh:`100926`.) -* The :class:`ctypes.py_object` type now supports subscription, +* The :class:`~ctypes.py_object` type now supports subscription, making it a :term:`generic type`. (Contributed by Brian Schubert in :gh:`132168`.) -* :mod:`ctypes` now supports :term:`free-threading builds `. +* :mod:`!ctypes` now supports :term:`free-threading builds `. (Contributed by Kumar Aditya and Peter Bierma in :gh:`127945`.) + curses ------ * Add the :func:`~curses.assume_default_colors` function, a refinement of the :func:`~curses.use_default_colors` function which - allows to change the color pair ``0``. + allows changing the color pair ``0``. (Contributed by Serhiy Storchaka in :gh:`133139`.) + datetime -------- -* Add :meth:`datetime.time.strptime` and :meth:`datetime.date.strptime`. +* Add the :meth:`~datetime.date.strptime` method to the + :class:`datetime.date` and :class:`datetime.time` classes. (Contributed by Wannes Boeykens in :gh:`41431`.) + decimal ------- -* Add alternative :class:`~decimal.Decimal` constructor - :meth:`Decimal.from_number() `. +* Add :meth:`.Decimal.from_number` as an alternative constructor for + :class:`~decimal.Decimal`. (Contributed by Serhiy Storchaka in :gh:`121798`.) -* Expose :func:`decimal.IEEEContext` to support creation of contexts +* Expose :func:`~decimal.IEEEContext` to support creation of contexts corresponding to the IEEE 754 (2008) decimal interchange formats. (Contributed by Sergey B Kirpichev in :gh:`53032`.) + difflib ------- * Comparison pages with highlighted changes generated by the - :class:`difflib.HtmlDiff` class now support dark mode. + :class:`~difflib.HtmlDiff` class now support 'dark mode'. (Contributed by Jiahao Li in :gh:`129939`.) + dis --- @@ -1486,7 +1321,7 @@ dis errno ----- -* Add :data:`errno.EHWPOISON` error code. +* Add the :data:`~errno.EHWPOISON` error code constant. (Contributed by James Roy in :gh:`126585`.) @@ -1494,39 +1329,43 @@ faulthandler ------------ * Add support for printing the C stack trace on systems that - :ref:`support it ` via :func:`faulthandler.dump_c_stack` - or via the *c_stack* argument in :func:`faulthandler.enable`. + :ref:`support it ` via the new + :func:`~faulthandler.dump_c_stack` function or via the *c_stack* argument + in :func:`faulthandler.enable`. (Contributed by Peter Bierma in :gh:`127604`.) fnmatch ------- -* Added :func:`fnmatch.filterfalse` for excluding names matching a pattern. +* Add :func:`~fnmatch.filterfalse`, a function to reject names + matching a given pattern. (Contributed by Bénédikt Tran in :gh:`74598`.) fractions --------- -* Add support for converting any objects that have the - :meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. +* A :class:`~fractions.Fraction` object may now be constructed from any + object with the :meth:`!as_integer_ratio` method. (Contributed by Serhiy Storchaka in :gh:`82017`.) -* Add alternative :class:`~fractions.Fraction` constructor - :meth:`Fraction.from_number() `. +* Add :meth:`.Fraction.from_number` as an alternative constructor for + :class:`~fractions.Fraction`. (Contributed by Serhiy Storchaka in :gh:`121797`.) functools --------- -* Add support to :func:`functools.partial` and - :func:`functools.partialmethod` for :data:`functools.Placeholder` sentinels - to reserve a place for positional arguments. +* Add the :data:`~functools.Placeholder` sentinel. + This may be used with the :func:`~functools.partial` + or :func:`~functools.partialmethod` functions to reserve a place + for positional arguments in the returned :ref:`partial object + `. (Contributed by Dominykas Grigonis in :gh:`119127`.) -* Allow the *initial* parameter of :func:`functools.reduce` to be passed +* Allow the *initial* parameter of :func:`~functools.reduce` to be passed as a keyword argument. (Contributed by Sayandip Dutta in :gh:`125916`.) @@ -1544,16 +1383,17 @@ getopt getpass ------- -* Support keyboard feedback by :func:`getpass.getpass` via the keyword-only - optional argument ``echo_char``. Placeholder characters are rendered whenever - a character is entered, and removed when a character is deleted. +* Support keyboard feedback in the :func:`~getpass.getpass` function via + the keyword-only optional argument *echo_char*. + Placeholder characters are rendered whenever a character is entered, + and removed when a character is deleted. (Contributed by Semyon Moroz in :gh:`77065`.) graphlib -------- -* Allow :meth:`graphlib.TopologicalSorter.prepare` to be called more than once +* Allow :meth:`.TopologicalSorter.prepare` to be called more than once as long as sorting has not started. (Contributed by Daniel Pope in :gh:`130914`.) @@ -1561,13 +1401,14 @@ graphlib heapq ----- -* Add functions for working with max-heaps: +* The :mod:`!heapq` module has improved support for working with max-heaps, + via the following new functions: - * :func:`heapq.heapify_max`, - * :func:`heapq.heappush_max`, - * :func:`heapq.heappop_max`, - * :func:`heapq.heapreplace_max` - * :func:`heapq.heappushpop_max` + * :func:`~heapq.heapify_max` + * :func:`~heapq.heappush_max` + * :func:`~heapq.heappop_max` + * :func:`~heapq.heapreplace_max` + * :func:`~heapq.heappushpop_max` hmac @@ -1575,6 +1416,8 @@ hmac * Add a built-in implementation for HMAC (:rfc:`2104`) using formally verified code from the `HACL* `__ project. + This implementation is used as a fallback when the OpenSSL implementation + of HMAC is not available. (Contributed by Bénédikt Tran in :gh:`99108`.) @@ -1590,9 +1433,12 @@ http the command-line interface (``python -m http.server``) through the following options: - * ``--tls-cert ``: Path to the TLS certificate file. - * ``--tls-key ``: Optional path to the private key file. - * ``--tls-password-file ``: Optional path to the password file for the private key. + * :option:`--tls-cert \ `: + Path to the TLS certificate file. + * :option:`--tls-key \ `: + Optional path to the private key file. + * :option:`--tls-password-file \ `: + Optional path to the password file for the private key. (Contributed by Semyon Moroz in :gh:`85162`.) @@ -1600,7 +1446,7 @@ http imaplib ------- -* Add :meth:`IMAP4.idle() `, implementing the IMAP4 +* Add :meth:`.IMAP4.idle`, implementing the IMAP4 ``IDLE`` command as defined in :rfc:`2177`. (Contributed by Forest in :gh:`55454`.) @@ -1608,15 +1454,16 @@ imaplib inspect ------- -* :func:`inspect.signature` takes a new argument *annotation_format* to control +* :func:`~inspect.signature` takes a new argument *annotation_format* to control the :class:`annotationlib.Format` used for representing annotations. (Contributed by Jelle Zijlstra in :gh:`101552`.) -* :meth:`inspect.Signature.format` takes a new argument *unquote_annotations*. - If true, string :term:`annotations ` are displayed without surrounding quotes. +* :meth:`.Signature.format` takes a new argument *unquote_annotations*. + If true, string :term:`annotations ` are displayed without + surrounding quotes. (Contributed by Jelle Zijlstra in :gh:`101552`.) -* Add function :func:`inspect.ispackage` to determine whether an object is a +* Add function :func:`~inspect.ispackage` to determine whether an object is a :term:`package` or not. (Contributed by Zhikang Yan in :gh:`125634`.) @@ -1628,7 +1475,7 @@ io :exc:`BlockingIOError` if the operation cannot immediately return bytes. (Contributed by Giovanni Siragusa in :gh:`109523`.) -* Add protocols :class:`io.Reader` and :class:`io.Writer` as simpler +* Add the :class:`~io.Reader` and :class:`~io.Writer` protocols as simpler alternatives to the pseudo-protocols :class:`typing.IO`, :class:`typing.TextIO`, and :class:`typing.BinaryIO`. (Contributed by Sebastian Rittau in :gh:`127648`.) @@ -1637,35 +1484,36 @@ io json ---- -* Add notes for JSON serialization errors that allow to identify the source - of the error. +* Add exception notes for JSON serialization errors that allow + identifying the source of the error. (Contributed by Serhiy Storchaka in :gh:`122163`.) -* Enable the :mod:`json` module to work as a script using the :option:`-m` - switch: :program:`python -m json`. +* Allow using the :mod:`json` module as a script using the :option:`-m` switch: + :program:`python -m json`. + This is now preferred to :program:`python -m json.tool`, + which is :term:`soft deprecated`. See the :ref:`JSON command-line interface ` documentation. (Contributed by Trey Hunner in :gh:`122873`.) - .. _whatsnew314-color-json: - * By default, the output of the :ref:`JSON command-line interface ` is highlighted in color. This can be controlled by :ref:`environment variables `. (Contributed by Tomas Roun in :gh:`131952`.) + linecache --------- -* :func:`linecache.getline` can retrieve source code for frozen modules. +* :func:`~linecache.getline` can now retrieve source code for frozen modules. (Contributed by Tian Gao in :gh:`131638`.) logging.handlers ---------------- -* :class:`logging.handlers.QueueListener` now implements the context - manager protocol, allowing it to be used in a :keyword:`with` statement. +* :class:`~logging.handlers.QueueListener` objects now support the + :term:`context manager` protocol. (Contributed by Charles Machalow in :gh:`132106`.) * :meth:`QueueListener.start ` now @@ -1683,14 +1531,13 @@ math mimetypes --------- -* Document the command-line for :mod:`mimetypes`. - It now exits with ``1`` on failure instead of ``0`` - and ``2`` on incorrect command-line parameters instead of ``1``. - Also, errors are printed to stderr instead of stdout and their text is made - tighter. +* Add a public :ref:`command-line ` for the module, + invoked via :program:`python -m mimetypes`. (Contributed by Oleg Iarygin and Hugo van Kemenade in :gh:`93096`.) -* Add MS and :rfc:`8081` MIME types for fonts: +* Add several new MIME types based on RFCs and common usage: + + .. rubric:: Microsoft and :rfc:`8081` MIME types for fonts * Embedded OpenType: ``application/vnd.ms-fontobject`` * OpenType Layout (OTF) ``font/otf`` @@ -1698,18 +1545,14 @@ mimetypes * WOFF 1.0 ``font/woff`` * WOFF 2.0 ``font/woff2`` - (Contributed by Sahil Prajapati and Hugo van Kemenade in :gh:`84852`.) - -* Add :rfc:`9559` MIME types for Matroska audiovisual data container - structures, containing: + .. rubric:: :rfc:`9559` MIME types for Matroska audiovisual + data container structures * audio with no video: ``audio/matroska`` (``.mka``) * video: ``video/matroska`` (``.mkv``) * stereoscopic video: ``video/matroska-3d`` (``.mk3d``) - (Contributed by Hugo van Kemenade in :gh:`89416`.) - -* Add MIME types for images with RFCs: + .. rubric:: Images with RFCs * :rfc:`1494`: CCITT Group 3 (``.g3``) * :rfc:`3362`: Real-time Facsimile, T.38 (``.t38``) @@ -1718,9 +1561,7 @@ mimetypes * :rfc:`4047`: Flexible Image Transport System (``.fits``) * :rfc:`7903`: Enhanced Metafile (``.emf``) and Windows Metafile (``.wmf``) - (Contributed by Hugo van Kemenade in :gh:`85957`.) - -* More MIME type changes: + .. rubric:: Other MIME type additions and changes * :rfc:`2361`: Change type for ``.avi`` to ``video/vnd.avi`` and for ``.wav`` to ``audio/vnd.wave`` @@ -1728,6 +1569,8 @@ mimetypes * :rfc:`5334`: Add Ogg media (``.oga``, ``.ogg`` and ``.ogx``) * :rfc:`6713`: Add gzip ``application/gzip`` (``.gz``) * :rfc:`9639`: Add FLAC ``audio/flac`` (``.flac``) + * :rfc:`9512` ``application/yaml`` MIME type for YAML files (``.yaml`` + and ``.yml``) * Add 7z ``application/x-7z-compressed`` (``.7z``) * Add Android Package ``application/vnd.android.package-archive`` (``.apk``) when not strict @@ -1750,11 +1593,9 @@ mimetypes * `W3C `__: Add EPUB ``application/epub+zip`` (``.epub``) - (Contributed by Hugo van Kemenade in :gh:`129965`.) - -* Add :rfc:`9512` ``application/yaml`` MIME type for YAML files (``.yaml`` - and ``.yml``). (Contributed by Sasha "Nelie" Chernykh and Hugo van Kemenade - in :gh:`132056`.) + (Contributed by Sahil Prajapati and Hugo van Kemenade in :gh:`84852`, + by Sasha "Nelie" Chernykh and Hugo van Kemenade in :gh:`132056`, + and by Hugo van Kemenade in :gh:`89416`, :gh:`85957`, and :gh:`129965`.) multiprocessing @@ -1762,14 +1603,16 @@ multiprocessing .. _whatsnew314-multiprocessing-start-method: -* The default :ref:`start method ` changed - from :ref:`fork ` to :ref:`forkserver - ` on platforms other than macOS and - Windows where it was already :ref:`spawn `. +* On Unix platforms other than macOS, :ref:`'forkserver' + ` is now the default :ref:`start + method ` + (replacing :ref:`'fork' `). + This change does not affect Windows or macOS, where :ref:`'spawn' + ` remains the default start method. If the threading incompatible *fork* method is required, you must explicitly - request it via a context from :func:`multiprocessing.get_context` (preferred) - or change the default via :func:`multiprocessing.set_start_method`. + request it via a context from :func:`~multiprocessing.get_context` (preferred) + or change the default via :func:`~multiprocessing.set_start_method`. See :ref:`forkserver restrictions ` for information and differences with the *fork* method and how this change @@ -1778,7 +1621,7 @@ multiprocessing (Contributed by Gregory P. Smith in :gh:`84559`.) -* :mod:`multiprocessing`'s ``"forkserver"`` start method now authenticates +* :mod:`multiprocessing`'s ``'forkserver'`` start method now authenticates its control socket to avoid solely relying on filesystem permissions to restrict what other processes could cause the forkserver to spawn workers and run code. @@ -1794,20 +1637,22 @@ multiprocessing (Contributed by Roy Hyunjin Han for :gh:`103134`.) * Add support for shared :class:`set` objects via - :meth:`SyncManager.set() `. - The :func:`set` in :func:`multiprocessing.Manager` method is now available. + :meth:`.SyncManager.set`. + The :func:`set` in :func:`~multiprocessing.Manager` method is now available. (Contributed by Mingyu Park in :gh:`129949`.) -* Add :func:`multiprocessing.Process.interrupt` which terminates the child +* Add the :meth:`~multiprocessing.Process.interrupt` + to :class:`multiprocessing.Process` objects, which terminates the child process by sending :py:const:`~signal.SIGINT`. This enables :keyword:`finally` clauses to print a stack trace for the terminated process. (Contributed by Artem Pulkin in :gh:`131913`.) + operator -------- -* Two new functions :func:`operator.is_none` and :func:`operator.is_not_none` - have been added, such that ``operator.is_none(obj)`` is equivalent +* Add :func:`~operator.is_none` and :func:`~operator.is_not_none` as a pair + of functions, such that ``operator.is_none(obj)`` is equivalent to ``obj is None`` and ``operator.is_not_none(obj)`` is equivalent to ``obj is not None``. (Contributed by Raymond Hettinger and Nico Mexis in :gh:`115808`.) @@ -1816,17 +1661,17 @@ operator os -- -* Add the :func:`os.reload_environ` function to update :data:`os.environ` and +* Add the :func:`~os.reload_environ` function to update :data:`os.environ` and :data:`os.environb` with changes to the environment made by :func:`os.putenv`, by :func:`os.unsetenv`, or made outside Python in the same process. (Contributed by Victor Stinner in :gh:`120057`.) * Add the :data:`~os.SCHED_DEADLINE` and :data:`~os.SCHED_NORMAL` constants - to the :mod:`os` module. + to the :mod:`!os` module. (Contributed by James Roy in :gh:`127688`.) -* Add the :func:`os.readinto` function to read into a +* Add the :func:`~os.readinto` function to read into a :ref:`buffer object ` from a file descriptor. (Contributed by Cody Maloney in :gh:`129205`.) @@ -1834,8 +1679,8 @@ os os.path ------- -* The *strict* parameter to :func:`os.path.realpath` accepts a new value, - :data:`os.path.ALLOW_MISSING`. +* The *strict* parameter to :func:`~os.path.realpath` accepts a new value, + :data:`~os.path.ALLOW_MISSING`. If used, errors other than :exc:`FileNotFoundError` will be re-raised; the resulting path can be missing but it will be free of symlinks. (Contributed by Petr Viktorin for :cve:`2025-4517`.) @@ -1854,8 +1699,8 @@ pathlib (Contributed by Barney Gale in :gh:`73991`.) -* Add :attr:`pathlib.Path.info` attribute, which stores an object - implementing the :class:`pathlib.types.PathInfo` protocol (also new). The +* Add the :attr:`~pathlib.Path.info` attribute, which stores an object + implementing the new :class:`pathlib.types.PathInfo` protocol. The object supports querying the file type and internally caching :func:`~os.stat` results. Path objects generated by :meth:`~pathlib.Path.iterdir` are initialized with file type information @@ -1866,7 +1711,26 @@ pathlib pdb --- -* Hardcoded breakpoints (:func:`breakpoint` and :func:`pdb.set_trace`) now +* The :mod:`pdb` module now supports remote attaching to a running Python process + using a new :option:`-p PID ` command-line option: + + .. code-block:: sh + + python -m pdb -p 1234 + + This will connect to the Python process with the given PID and allow you to + debug it interactively. Notice that due to how the Python interpreter works + attaching to a remote process that is blocked in a system call or waiting for + I/O will only work once the next bytecode instruction is executed or when the + process receives a signal. + + This feature uses :ref:`PEP 768 ` + and the new :func:`sys.remote_exec` function to attach to the remote process + and send the PDB commands to it. + + (Contributed by Matt Wozniski and Pablo Galindo in :gh:`131591`.) + +* Hardcoded breakpoints (:func:`breakpoint` and :func:`~pdb.set_trace`) now reuse the most recent :class:`~pdb.Pdb` instance that calls :meth:`~pdb.Pdb.set_trace`, instead of creating a new one each time. As a result, all the instance specific data like :pdbcmd:`display` and @@ -1905,8 +1769,8 @@ pdb (Contributed by Tian Gao in :gh:`132576`.) * Source code displayed in :mod:`pdb` will be syntax-highlighted. This feature - can be controlled using the same methods as PyREPL, in addition to the newly - added ``colorize`` argument of :class:`pdb.Pdb`. + can be controlled using the same methods as the default :term:`interactive` + shell, in addition to the newly added ``colorize`` argument of :class:`pdb.Pdb`. (Contributed by Tian Gao and Łukasz Langa in :gh:`133355`.) @@ -1916,15 +1780,16 @@ pickle * Set the default protocol version on the :mod:`pickle` module to 5. For more details, see :ref:`pickle protocols `. -* Add notes for pickle serialization errors that allow to identify the source - of the error. +* Add exception notes for pickle serialization errors that allow + identifying the source of the error. (Contributed by Serhiy Storchaka in :gh:`122213`.) platform -------- -* Add :func:`platform.invalidate_caches` to invalidate the cached results. +* Add :func:`~platform.invalidate_caches`, a function to invalidate + cached results in the :mod:`!platform` module. (Contributed by Bénédikt Tran in :gh:`122549`.) @@ -1936,6 +1801,19 @@ pydoc (Contributed by Jelle Zijlstra in :gh:`101552`.) +re +-- + +* Support ``\z`` as a synonym for ``\Z`` in :mod:`regular expressions `. + It is interpreted unambiguously in many other regular expression engines, + unlike ``\Z``, which has subtly different behavior. + (Contributed by Serhiy Storchaka in :gh:`133306`.) + +* ``\B`` in :mod:`regular expression ` now matches the empty input string, + meaning that it is now always the opposite of ``\b``. + (Contributed by Serhiy Storchaka in :gh:`124130`.) + + socket ------ @@ -1962,11 +1840,12 @@ socket * Add many new constants. (Contributed by Serhiy Storchaka in :gh:`132734`.) + ssl --- -* Indicate through :data:`ssl.HAS_PHA` whether the :mod:`ssl` module supports - TLSv1.3 post-handshake client authentication (PHA). +* Indicate through the :data:`~ssl.HAS_PHA` Boolean whether the :mod:`!ssl` + module supports TLSv1.3 post-handshake client authentication (PHA). (Contributed by Will Childs-Klein in :gh:`128036`.) @@ -1982,7 +1861,7 @@ struct symtable -------- -* Expose the following :class:`symtable.Symbol` methods: +* Expose the following :class:`~symtable.Symbol` methods: * :meth:`~symtable.Symbol.is_comp_cell` * :meth:`~symtable.Symbol.is_comp_iter` @@ -1997,28 +1876,41 @@ sys * The previously undocumented special function :func:`sys.getobjects`, which only exists in specialized builds of Python, may now return objects from other interpreters than the one it's called in. + (Contributed by Eric Snow in :gh:`125286`.) * Add :func:`sys._is_immortal` for determining if an object is :term:`immortal`. (Contributed by Peter Bierma in :gh:`128509`.) -* On FreeBSD, :data:`sys.platform` doesn't contain the major version anymore. +* On FreeBSD, :data:`sys.platform` no longer contains the major version number. It is always ``'freebsd'``, instead of ``'freebsd13'`` or ``'freebsd14'``. + (Contributed by Michael Osipov in :gh:`129393`.) * Raise :exc:`DeprecationWarning` for :func:`sys._clear_type_cache`. This function was deprecated in Python 3.13 but it didn't raise a runtime warning. +* Add :func:`sys.remote_exec` to implement the new external debugger interface. + See :ref:`PEP 768 ` for details. + (Contributed by Pablo Galindo Salgado, Matt Wozniski, and Ivona Stojanovic + in :gh:`131591`.) + +* Add the :data:`sys._jit` namespace, containing utilities for introspecting + just-in-time compilation. + (Contributed by Brandt Bucher in :gh:`133231`.) + sys.monitoring -------------- -* Two new events are added: :monitoring-event:`BRANCH_LEFT` and - :monitoring-event:`BRANCH_RIGHT`. The ``BRANCH`` event is deprecated. +* Add two new monitoring events, :monitoring-event:`BRANCH_LEFT` and + :monitoring-event:`BRANCH_RIGHT`. + These replace and deprecate the :monitoring-event:`!BRANCH` event. + (Contributed by Mark Shannon in :gh:`122548`.) sysconfig --------- -* Add ``ABIFLAGS`` key to :func:`sysconfig.get_config_vars` on Windows. +* Add ``ABIFLAGS`` key to :func:`~sysconfig.get_config_vars` on Windows. (Contributed by Xuehai Pan in :gh:`131799`.) @@ -2028,15 +1920,18 @@ tarfile * :func:`~tarfile.data_filter` now normalizes symbolic link targets in order to avoid path traversal attacks. (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2025-4138`.) + * :func:`~tarfile.TarFile.extractall` now skips fixing up directory attributes when a directory was removed or replaced by another kind of file. (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2024-12718`.) + * :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` now (re-)apply the extraction filter when substituting a link (hard or symbolic) with a copy of another archive member, and when fixing up directory attributes. The former raises a new exception, :exc:`~tarfile.LinkFallbackError`. (Contributed by Petr Viktorin for :cve:`2025-4330` and :cve:`2024-12718`.) + * :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` no longer extract rejected members when :func:`~tarfile.TarFile.errorlevel` is zero. @@ -2056,17 +1951,18 @@ tkinter ------- * Make :mod:`tkinter` widget methods :meth:`!after` and :meth:`!after_idle` - accept arguments passed by keyword. + accept keyword arguments. (Contributed by Zhikang Yan in :gh:`126899`.) -* Add ability to specify name for :class:`!tkinter.OptionMenu` and +* Add ability to specify a name for :class:`!tkinter.OptionMenu` and :class:`!tkinter.ttk.OptionMenu`. (Contributed by Zhikang Yan in :gh:`130482`.) + turtle ------ -* Add context managers for :func:`turtle.fill`, :func:`turtle.poly` +* Add context managers for :func:`turtle.fill`, :func:`turtle.poly`, and :func:`turtle.no_animation`. (Contributed by Marie Roald and Yngve Mardal Moe in :gh:`126350`.) @@ -2084,44 +1980,59 @@ typing .. _whatsnew314-typing-union: -* :class:`types.UnionType` and :class:`typing.Union` are now aliases for each other, - meaning that both old-style unions (created with ``Union[int, str]``) and new-style - unions (``int | str``) now create instances of the same runtime type. This unifies - the behavior between the two syntaxes, but leads to some differences in behavior that +* The :class:`types.UnionType` and :class:`typing.Union` types are now + aliases for each other, meaning that both old-style unions + (created with ``Union[int, str]``) and new-style unions (``int | str``) + now create instances of the same runtime type. This unifies the behavior + between the two syntaxes, but leads to some differences in behavior that may affect users who introspect types at runtime: - - Both syntaxes for creating a union now produce the same string representation in - ``repr()``. For example, ``repr(Union[int, str])`` - is now ``"int | str"`` instead of ``"typing.Union[int, str]"``. - - Unions created using the old syntax are no longer cached. Previously, running - ``Union[int, str]`` multiple times would return the same object - (``Union[int, str] is Union[int, str]`` would be ``True``), but now it will - return two different objects. Users should use ``==`` to compare unions for equality, not - ``is``. New-style unions have never been cached this way. - This change could increase memory usage for some programs that use a large number of - unions created by subscripting ``typing.Union``. However, several factors offset this cost: - unions used in annotations are no longer evaluated by default in Python 3.14 - because of :pep:`649`; an instance of :class:`types.UnionType` is - itself much smaller than the object returned by ``Union[]`` was on prior Python - versions; and removing the cache also saves some space. It is therefore - unlikely that this change will cause a significant increase in memory usage for most - users. + - Both syntaxes for creating a union now produce the same string + representation in :func:`repr`. + For example, ``repr(Union[int, str])`` is now ``"int | str"`` instead of + ``"typing.Union[int, str]"``. + + - Unions created using the old syntax are no longer cached. + Previously, running ``Union[int, str]`` multiple times would return + the same object (``Union[int, str] is Union[int, str]`` would be ``True``), + but now it will return two different objects. + Use ``==`` to compare unions for equality, not ``is``. + New-style unions have never been cached this way. + This change could increase memory usage for some programs that use + a large number of unions created by subscripting ``typing.Union``. + However, several factors offset this cost: + unions used in annotations are no longer evaluated by default in Python + 3.14 because of :pep:`649`; an instance of :class:`types.UnionType` is + itself much smaller than the object returned by ``Union[]`` was on prior + Python versions; and removing the cache also saves some space. + It is therefore unlikely that this change will cause a significant increase + in memory usage for most users. + - Previously, old-style unions were implemented using the private class - ``typing._UnionGenericAlias``. This class is no longer needed for the implementation, - but it has been retained for backward compatibility, with removal scheduled for Python - 3.17. Users should use documented introspection helpers like :func:`typing.get_origin` - and :func:`typing.get_args` instead of relying on private implementation details. - - It is now possible to use :class:`typing.Union` itself in :func:`isinstance` checks. - For example, ``isinstance(int | str, typing.Union)`` will return ``True``; previously - this raised :exc:`TypeError`. - - The ``__args__`` attribute of :class:`typing.Union` objects is no longer writable. - - It is no longer possible to set any attributes on :class:`typing.Union` objects. + ``typing._UnionGenericAlias``. + This class is no longer needed for the implementation, + but it has been retained for backward compatibility, + with removal scheduled for Python 3.17. + Users should use documented introspection helpers like + :func:`~typing.get_origin` and :func:`typing.get_args` instead of + relying on private implementation details. + + - It is now possible to use :class:`typing.Union` itself in + :func:`isinstance` checks. + For example, ``isinstance(int | str, typing.Union)`` will return ``True``; + previously this raised :exc:`TypeError`. + + - The :attr:`!__args__` attribute of :class:`typing.Union` objects is + no longer writable. + + - It is no longer possible to set any attributes on :class:`~typing.Union` + objects. This only ever worked for dunder attributes on previous versions, was never documented to work, and was subtly broken in many cases. (Contributed by Jelle Zijlstra in :gh:`105499`.) -* :class:`typing.TypeAliasType` now supports star unpacking. +* :class:`~typing.TypeAliasType` now supports star unpacking. unicodedata @@ -2130,11 +2041,11 @@ unicodedata * The Unicode database has been updated to Unicode 16.0.0. -.. _whatsnew314-color-unittest: - unittest -------- +.. _whatsnew314-color-unittest: + * :mod:`unittest` output is now colored by default. This can be controlled by :ref:`environment variables `. @@ -2172,7 +2083,7 @@ urllib * Improve ergonomics and standards compliance when parsing and emitting ``file:`` URLs. - In :func:`urllib.request.url2pathname`: + In :func:`~urllib.request.url2pathname`: - Accept a complete URL when the new *require_scheme* argument is set to true. @@ -2183,7 +2094,7 @@ urllib - Raise :exc:`~urllib.error.URLError` if a URL authority isn't local, except on Windows where we return a UNC path as before. - In :func:`urllib.request.pathname2url`: + In :func:`~urllib.request.pathname2url`: - Return a complete URL when the new *add_scheme* argument is set to true. - Include an empty URL authority when a path begins with a slash. For @@ -2199,16 +2110,17 @@ urllib uuid ---- -* Add support for UUID versions 6, 7, and 8 via :func:`uuid.uuid6`, - :func:`uuid.uuid7`, and :func:`uuid.uuid8` respectively, as specified +* Add support for UUID versions 6, 7, and 8 via :func:`~uuid.uuid6`, + :func:`~uuid.uuid7`, and :func:`~uuid.uuid8` respectively, as specified in :rfc:`9562`. (Contributed by Bénédikt Tran in :gh:`89083`.) -* :const:`uuid.NIL` and :const:`uuid.MAX` are now available to represent the +* :const:`~uuid.NIL` and :const:`~uuid.MAX` are now available to represent the Nil and Max UUID formats as defined by :rfc:`9562`. (Contributed by Nick Pope in :gh:`128427`.) -* Allow to generate multiple UUIDs at once via :option:`python -m uuid --count `. +* Allow generating multiple UUIDs simultaneously on the command-line via + :option:`python -m uuid --count `. (Contributed by Simon Legner in :gh:`131236`.) @@ -2226,14 +2138,13 @@ webbrowser zipfile ------- -* Added :func:`ZipInfo._for_archive ` +* Added :meth:`ZipInfo._for_archive `, a method to resolve suitable defaults for a :class:`~zipfile.ZipInfo` object as used by :func:`ZipFile.writestr `. (Contributed by Bénédikt Tran in :gh:`123424`.) -* :meth:`zipfile.ZipFile.writestr` now respects ``SOURCE_DATE_EPOCH`` that - distributions can set centrally and have build tools consume this in order - to produce reproducible output. +* :meth:`.ZipFile.writestr` now respects the :envvar:`SOURCE_DATE_EPOCH` + environment variable in order to better support reproducible builds. (Contributed by Jiahao Li in :gh:`91279`.) @@ -2253,12 +2164,18 @@ Optimizations (Contributed by Adam Turner, Bénédikt Tran, Chris Markiewicz, Eli Schwartz, Hugo van Kemenade, Jelle Zijlstra, and others in :gh:`118761`.) +* The interpreter now avoids some reference count modifications internally + when it's safe to do so. + This can lead to different values being returned from :func:`sys.getrefcount` + and :c:func:`Py_REFCNT` compared to previous versions of Python. + See :ref:`below ` for details. + asyncio ------- -* Standard benchmark results have improved by 10-20%, following the - implementation of a new per-thread double linked list +* Standard benchmark results have improved by 10-20% following the + implementation of a new per-thread doubly linked list for :class:`native tasks `, also reducing memory usage. This enables external introspection tools such as @@ -2300,7 +2217,18 @@ difflib gc -- -* The new :ref:`incremental garbage collector ` +* **From Python 3.14.5 onwards:** + + Python 3.14.0-3.14.4 shipped with a new incremental garbage collector. + However, due to a number of `reports + `__ + of significant memory pressure in production environments, + it has been reverted back to the generational GC from 3.13. + This is the GC now used in Python 3.14.5 and later. + +* **Previously in Python 3.14.0-3.14.4:** + + The new :ref:`incremental garbage collector ` means that maximum pause times are reduced by an order of magnitude or more for larger heaps. @@ -2361,6 +2289,15 @@ pdb (Contributed by Tian Gao in :gh:`124533`.) +textwrap +-------- + +* Optimize the :func:`~textwrap.dedent` function, improving performance by + an average of 2.4x, with larger improvements for bigger inputs, + and fix a bug with incomplete normalization of blank lines with whitespace + characters other than space and tab. + + uuid ---- @@ -2561,14 +2498,6 @@ asyncio runner.run(operation_two()) -collections.abc ---------------- - -* Remove :class:`!ByteString`, which has been deprecated since Python 3.12. - (Contributed by Nikita Sobolev in :gh:`118803`.) - - - email ----- @@ -2647,13 +2576,6 @@ sqlite3 (Contributed by Erlend E. Aasland in :gh:`118928` and :gh:`101693`.) -typing ------- - -* Remove :class:`!ByteString`, which has been deprecated since Python 3.12. - (Contributed by Nikita Sobolev in :gh:`118803`.) - - urllib ------ @@ -2676,25 +2598,34 @@ urllib Deprecated ========== +New deprecations +---------------- + +* Passing a complex number as the *real* or *imag* argument in the + :func:`complex` constructor is now deprecated; + complex numbers should only be passed as a single positional argument. + (Contributed by Serhiy Storchaka in :gh:`109218`.) + * :mod:`argparse`: - * Passing the undocumented keyword argument *prefix_chars* to - :meth:`~argparse.ArgumentParser.add_argument_group` is now - deprecated. + * Passing the undocumented keyword argument *prefix_chars* to the + :meth:`~argparse.ArgumentParser.add_argument_group` method is now deprecated. (Contributed by Savannah Ostrowski in :gh:`125563`.) + * Deprecated the :class:`argparse.FileType` type converter. - Anything with resource management should be done downstream after the - arguments are parsed. + Anything relating to resource management should be handled + downstream, after the arguments have been parsed. (Contributed by Serhiy Storchaka in :gh:`58032`.) * :mod:`asyncio`: - * :func:`!asyncio.iscoroutinefunction` is deprecated + * The :func:`!asyncio.iscoroutinefunction` is now deprecated and will be removed in Python 3.16; use :func:`inspect.iscoroutinefunction` instead. (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) - * :mod:`asyncio` policy system is deprecated and will be removed in Python 3.16. + * The :mod:`asyncio` policy system is deprecated + and will be removed in Python 3.16. In particular, the following classes and functions are deprecated: * :class:`asyncio.AbstractEventLoopPolicy` @@ -2705,99 +2636,102 @@ Deprecated * :func:`asyncio.set_event_loop_policy` Users should use :func:`asyncio.run` or :class:`asyncio.Runner` with - *loop_factory* to use the desired event loop implementation. + the *loop_factory* argument to use the desired event loop implementation. - For example, to use :class:`asyncio.SelectorEventLoop` on Windows:: + For example, to use :class:`asyncio.SelectorEventLoop` on Windows: - import asyncio + .. code-block:: python - async def main(): - ... + import asyncio - asyncio.run(main(), loop_factory=asyncio.SelectorEventLoop) + async def main(): + ... + + asyncio.run(main(), loop_factory=asyncio.SelectorEventLoop) (Contributed by Kumar Aditya in :gh:`127949`.) -* :mod:`builtins`: - Passing a complex number as the *real* or *imag* argument in the - :func:`complex` constructor is now deprecated; it should only be passed - as a single positional argument. - (Contributed by Serhiy Storchaka in :gh:`109218`.) - * :mod:`codecs`: - :func:`codecs.open` is now deprecated. Use :func:`open` instead. + The :func:`codecs.open` function is now deprecated, + and will be removed in a future version of Python. + Use :func:`open` instead. (Contributed by Inada Naoki in :gh:`133036`.) * :mod:`ctypes`: * On non-Windows platforms, setting :attr:`.Structure._pack_` to use a - MSVC-compatible default memory layout is deprecated in favor of setting - :attr:`.Structure._layout_` to ``'ms'``. + MSVC-compatible default memory layout is now deprecated in favor of setting + :attr:`.Structure._layout_` to ``'ms'``, and will be removed in Python 3.19. (Contributed by Petr Viktorin in :gh:`131747`.) - * Calling :func:`ctypes.POINTER` on a string is deprecated. - Use :ref:`ctypes-incomplete-types` for self-referential structures. + * Calling :func:`ctypes.POINTER` on a string is now deprecated. + Use :ref:`incomplete types ` + for self-referential structures. Also, the internal ``ctypes._pointer_type_cache`` is deprecated. See :func:`ctypes.POINTER` for updated implementation details. (Contributed by Sergey Myrianov in :gh:`100926`.) * :mod:`functools`: Calling the Python implementation of :func:`functools.reduce` with *function* - or *sequence* as keyword arguments is now deprecated. + or *sequence* as keyword arguments is now deprecated; + the parameters will be made positional-only in Python 3.16. (Contributed by Kirill Podoprigora in :gh:`121676`.) * :mod:`logging`: - Support for custom logging handlers with the *strm* argument is deprecated - and scheduled for removal in Python 3.16. Define handlers with the *stream* - argument instead. (Contributed by Mariusz Felisiak in :gh:`115032`.) + Support for custom logging handlers with the *strm* argument + is now deprecated and scheduled for removal in Python 3.16. + Define handlers with the *stream* argument instead. + (Contributed by Mariusz Felisiak in :gh:`115032`.) * :mod:`mimetypes`: - Valid extensions start with a '.' or are empty for + Valid extensions are either empty or must start with '.' for :meth:`mimetypes.MimeTypes.add_type`. Undotted extensions are deprecated and will raise a :exc:`ValueError` in Python 3.16. (Contributed by Hugo van Kemenade in :gh:`75223`.) -* :mod:`!nturl2path`: This module is now deprecated. Call - :func:`urllib.request.url2pathname` and :func:`~urllib.request.pathname2url` - instead. +* :mod:`!nturl2path`: + This module is now deprecated. Call :func:`urllib.request.url2pathname` + and :func:`~urllib.request.pathname2url` instead. (Contributed by Barney Gale in :gh:`125866`.) * :mod:`os`: - :term:`Soft deprecate ` :func:`os.popen` and - :func:`os.spawn* ` functions. They should no longer be used to - write new code. The :mod:`subprocess` module is recommended instead. + The :func:`os.popen` and :func:`os.spawn* ` functions + are now :term:`soft deprecated`. + They should no longer be used to write new code. + The :mod:`subprocess` module is recommended instead. (Contributed by Victor Stinner in :gh:`120743`.) * :mod:`pathlib`: - :meth:`!pathlib.PurePath.as_uri` is deprecated and will be removed in Python - 3.19. Use :meth:`pathlib.Path.as_uri` instead. + :meth:`!pathlib.PurePath.as_uri` is now deprecated + and scheduled for removal in Python 3.19. + Use :meth:`pathlib.Path.as_uri` instead. (Contributed by Barney Gale in :gh:`123599`.) * :mod:`pdb`: The undocumented ``pdb.Pdb.curframe_locals`` attribute is now a deprecated - read-only property. The low overhead dynamic frame locals access added in - Python 3.13 by PEP 667 means the frame locals cache reference previously - stored in this attribute is no longer needed. Derived debuggers should access + read-only property, which will be removed in a future version of Python. + The low overhead dynamic frame locals access added in Python 3.13 by :pep:`667` + means the frame locals cache reference previously stored in this attribute + is no longer needed. Derived debuggers should access ``pdb.Pdb.curframe.f_locals`` directly in Python 3.13 and later versions. (Contributed by Tian Gao in :gh:`124369` and :gh:`125951`.) * :mod:`symtable`: - Deprecate :meth:`symtable.Class.get_methods` due to the lack of interest. + Deprecate :meth:`symtable.Class.get_methods` due to the lack of interest, + scheduled for removal in Python 3.16. (Contributed by Bénédikt Tran in :gh:`119698`.) * :mod:`tkinter`: The :class:`!tkinter.Variable` methods :meth:`!trace_variable`, :meth:`!trace_vdelete` and :meth:`!trace_vinfo` are now deprecated. - Use :meth:`!trace_add`, :meth:`!trace_remove` and :meth:`!trace_info` - instead. + Use :meth:`!trace_add`, :meth:`!trace_remove` and :meth:`!trace_info` instead. (Contributed by Serhiy Storchaka in :gh:`120220`.) * :mod:`urllib.parse`: Accepting objects with false values (like ``0`` and ``[]``) except empty - strings, byte-like objects and ``None`` in :mod:`urllib.parse` functions - :func:`~urllib.parse.parse_qsl` and :func:`~urllib.parse.parse_qs` is now - deprecated. + strings, bytes-like objects and ``None`` in :func:`~urllib.parse.parse_qsl` + and :func:`~urllib.parse.parse_qs` is now deprecated. (Contributed by Serhiy Storchaka in :gh:`116897`.) .. Add deprecations above alphabetically, not here at the end. @@ -2808,78 +2742,145 @@ Deprecated .. include:: ../deprecations/pending-removal-in-3.17.rst +.. include:: ../deprecations/pending-removal-in-3.18.rst + .. include:: ../deprecations/pending-removal-in-3.19.rst +.. include:: ../deprecations/pending-removal-in-3.20.rst + .. include:: ../deprecations/pending-removal-in-future.rst CPython bytecode changes ======================== -* Replaced the opcode ``BINARY_SUBSCR`` by :opcode:`BINARY_OP` with oparg ``NB_SUBSCR``. +* Replaced the opcode :opcode:`!BINARY_SUBSCR` by the :opcode:`BINARY_OP` + opcode with the ``NB_SUBSCR`` oparg. (Contributed by Irit Katriel in :gh:`100239`.) +* Add the :opcode:`BUILD_INTERPOLATION` and :opcode:`BUILD_TEMPLATE` + opcodes to construct new :class:`~string.templatelib.Interpolation` + and :class:`~string.templatelib.Template` instances, respectively. + (Contributed by Lysandros Nikolaou and others in :gh:`132661`; + see also :ref:`PEP 750: Template strings `). + +* Remove the :opcode:`!BUILD_CONST_KEY_MAP` opcode. + Use :opcode:`BUILD_MAP` instead. + (Contributed by Mark Shannon in :gh:`122160`.) + +* Replace the :opcode:`!LOAD_ASSERTION_ERROR` opcode with + :opcode:`LOAD_COMMON_CONSTANT` and add support for loading + :exc:`NotImplementedError`. + +* Add the :opcode:`LOAD_FAST_BORROW` and :opcode:`LOAD_FAST_BORROW_LOAD_FAST_BORROW` + opcodes to reduce reference counting overhead when the interpreter can prove + that the reference in the frame outlives the reference loaded onto the stack. + (Contributed by Matt Page in :gh:`130704`.) + +* Add the :opcode:`LOAD_SMALL_INT` opcode, which pushes a small integer + equal to the ``oparg`` to the stack. + The :opcode:`!RETURN_CONST` opcode is removed as it is no longer used. + (Contributed by Mark Shannon in :gh:`125837`.) + +* Add the new :opcode:`LOAD_SPECIAL` instruction. + Generate code for :keyword:`with` and :keyword:`async with` statements + using the new instruction. + Removed the :opcode:`!BEFORE_WITH` and :opcode:`!BEFORE_ASYNC_WITH` instructions. + (Contributed by Mark Shannon in :gh:`120507`.) + +* Add the :opcode:`POP_ITER` opcode to support 'virtual' iterators. + (Contributed by Mark Shannon in :gh:`132554`.) + + +Pseudo-instructions +------------------- + +* Add the :opcode:`!ANNOTATIONS_PLACEHOLDER` pseudo instruction + to support partially executed module-level annotations with + :ref:`deferred evaluation of annotations `. + (Contributed by Jelle Zijlstra in :gh:`130907`.) + +* Add the :opcode:`!BINARY_OP_EXTEND` pseudo instruction, + which executes a pair of functions (guard and specialization functions) + accessed from the inline cache. + (Contributed by Irit Katriel in :gh:`100239`.) + +* Add three specializations for :opcode:`CALL_KW`; + :opcode:`!CALL_KW_PY` for calls to Python functions, + :opcode:`!CALL_KW_BOUND_METHOD` for calls to bound methods, and + :opcode:`!CALL_KW_NON_PY` for all other calls. + (Contributed by Mark Shannon in :gh:`118093`.) + +* Add the :opcode:`JUMP_IF_TRUE` and :opcode:`JUMP_IF_FALSE` pseudo instructions, + conditional jumps which do not impact the stack. + Replaced by the sequence ``COPY 1``, ``TO_BOOL``, ``POP_JUMP_IF_TRUE/FALSE``. + (Contributed by Irit Katriel in :gh:`124285`.) + +* Add the :opcode:`!LOAD_CONST_MORTAL` pseudo instruction. + (Contributed by Mark Shannon in :gh:`128685`.) + +* Add the :opcode:`!LOAD_CONST_IMMORTAL` pseudo instruction, + which does the same as :opcode:`!LOAD_CONST`, but is more efficient + for immortal objects. + (Contributed by Mark Shannon in :gh:`125837`.) + +* Add the :opcode:`NOT_TAKEN` pseudo instruction, used by :mod:`sys.monitoring` + to record branch events (such as :monitoring-event:`BRANCH_LEFT`). + (Contributed by Mark Shannon in :gh:`122548`.) + C API changes ============= -New features ------------- +.. _whatsnew314-capi-config: -* Add :c:func:`PyLong_GetSign` function to get the sign of :class:`int` objects. - (Contributed by Sergey B Kirpichev in :gh:`116560`.) +Python configuration C API +-------------------------- -* Add a new :c:type:`PyUnicodeWriter` API to create a Python :class:`str` - object: +Add a :ref:`PyInitConfig C API ` to configure the Python +initialization without relying on C structures and the ability to make +ABI-compatible changes in the future. - * :c:func:`PyUnicodeWriter_Create` - * :c:func:`PyUnicodeWriter_DecodeUTF8Stateful` - * :c:func:`PyUnicodeWriter_Discard` - * :c:func:`PyUnicodeWriter_Finish` - * :c:func:`PyUnicodeWriter_Format` - * :c:func:`PyUnicodeWriter_WriteASCII` - * :c:func:`PyUnicodeWriter_WriteChar` - * :c:func:`PyUnicodeWriter_WriteRepr` - * :c:func:`PyUnicodeWriter_WriteStr` - * :c:func:`PyUnicodeWriter_WriteSubstring` - * :c:func:`PyUnicodeWriter_WriteUCS4` - * :c:func:`PyUnicodeWriter_WriteUTF8` - * :c:func:`PyUnicodeWriter_WriteWideChar` +Complete the :pep:`587` :ref:`PyConfig C API ` by adding +:c:func:`PyInitConfig_AddModule` which can be used to add a built-in extension +module; a feature previously referred to as the "inittab". - (Contributed by Victor Stinner in :gh:`119182`.) +Add :c:func:`PyConfig_Get` and :c:func:`PyConfig_Set` functions to get and set +the current runtime configuration. -* Add :c:func:`PyIter_NextItem` to replace :c:func:`PyIter_Next`, - which has an ambiguous return value. - (Contributed by Irit Katriel and Erlend Aasland in :gh:`105201`.) +:pep:`587` 'Python Initialization Configuration' unified all the ways +to configure Python's initialization. This PEP also unifies the configuration +of Python's preinitialization and initialization in a single API. +Moreover, this PEP only provides a single choice to embed Python, +instead of having two 'Python' and 'Isolated' choices (PEP 587), +to further simplify the API. -* Add :c:func:`PyLong_IsPositive`, :c:func:`PyLong_IsNegative` - and :c:func:`PyLong_IsZero` for checking if :c:type:`PyLongObject` - is positive, negative, or zero, respectively. - (Contributed by James Roy and Sergey B Kirpichev in :gh:`126061`.) +The lower level PEP 587 PyConfig API remains available for use cases +with an intentionally higher level of coupling to CPython implementation details +(such as emulating the full functionality of CPython's CLI, including its +configuration mechanisms). -* Add new functions to convert C ```` numbers from/to Python - :class:`int`: +(Contributed by Victor Stinner in :gh:`107954`.) - * :c:func:`PyLong_AsInt32` - * :c:func:`PyLong_AsInt64` - * :c:func:`PyLong_AsUInt32` - * :c:func:`PyLong_AsUInt64` - * :c:func:`PyLong_FromInt32` - * :c:func:`PyLong_FromInt64` - * :c:func:`PyLong_FromUInt32` - * :c:func:`PyLong_FromUInt64` +.. seealso:: :pep:`741` and :pep:`587` - (Contributed by Victor Stinner in :gh:`120389`.) + +New features in the C API +------------------------- + +* Add :c:func:`Py_PACK_VERSION` and :c:func:`Py_PACK_FULL_VERSION`, + two new macros for bit-packing Python version numbers. + This is useful for comparisons with :c:var:`Py_Version` + or :c:macro:`PY_VERSION_HEX`. + (Contributed by Petr Viktorin in :gh:`128629`.) * Add :c:func:`PyBytes_Join(sep, iterable) ` function, similar to ``sep.join(iterable)`` in Python. (Contributed by Victor Stinner in :gh:`121645`.) -* Add :c:func:`Py_HashBuffer` to compute and return the hash value of a buffer. - (Contributed by Antoine Pitrou and Victor Stinner in :gh:`122854`.) - -* Add functions to get and set the current runtime Python configuration - (:pep:`741`): +* Add functions to manipulate the configuration of the current + runtime Python interpreter + (:ref:`PEP 741: Python configuration C API `): * :c:func:`PyConfig_Get` * :c:func:`PyConfig_GetInt` @@ -2888,7 +2889,8 @@ New features (Contributed by Victor Stinner in :gh:`107954`.) -* Add functions to configure the Python initialization (:pep:`741`): +* Add functions to configure Python initialization + (:ref:`PEP 741: Python configuration C API `): * :c:func:`Py_InitializeFromInitConfig` * :c:func:`PyInitConfig_AddModule` @@ -2907,7 +2909,50 @@ New features (Contributed by Victor Stinner in :gh:`107954`.) -* Add a new import and export API for Python :class:`int` objects (:pep:`757`): +* Add :c:func:`Py_fopen` function to open a file. + This works similarly to the standard C :c:func:`!fopen` function, + instead accepting a Python object for the *path* parameter + and setting an exception on error. + The corresponding new :c:func:`Py_fclose` function should be used + to close a file. + (Contributed by Victor Stinner in :gh:`127350`.) + +* Add :c:func:`Py_HashBuffer` to compute and return the hash value of a buffer. + (Contributed by Antoine Pitrou and Victor Stinner in :gh:`122854`.) + +* Add :c:func:`PyImport_ImportModuleAttr` and + :c:func:`PyImport_ImportModuleAttrString` helper functions to import a module + and get an attribute of the module. + (Contributed by Victor Stinner in :gh:`128911`.) + +* Add :c:func:`PyIter_NextItem` to replace :c:func:`PyIter_Next`, + which has an ambiguous return value. + (Contributed by Irit Katriel and Erlend Aasland in :gh:`105201`.) + +* Add :c:func:`PyLong_GetSign` function to get the sign of :class:`int` objects. + (Contributed by Sergey B Kirpichev in :gh:`116560`.) + +* Add :c:func:`PyLong_IsPositive`, :c:func:`PyLong_IsNegative` + and :c:func:`PyLong_IsZero` for checking if :c:type:`PyLongObject` + is positive, negative, or zero, respectively. + (Contributed by James Roy and Sergey B Kirpichev in :gh:`126061`.) + +* Add new functions to convert C ```` numbers to/from + Python :class:`int` objects: + + * :c:func:`PyLong_AsInt32` + * :c:func:`PyLong_AsInt64` + * :c:func:`PyLong_AsUInt32` + * :c:func:`PyLong_AsUInt64` + * :c:func:`PyLong_FromInt32` + * :c:func:`PyLong_FromInt64` + * :c:func:`PyLong_FromUInt32` + * :c:func:`PyLong_FromUInt64` + + (Contributed by Victor Stinner in :gh:`120389`.) + +* Add a new import and export API for Python :class:`int` objects + (:pep:`757`): * :c:func:`PyLong_GetNativeLayout` * :c:func:`PyLong_Export` @@ -2918,152 +2963,151 @@ New features (Contributed by Sergey B Kirpichev and Victor Stinner in :gh:`102471`.) -* Add :c:func:`PyType_GetBaseByToken` and :c:data:`Py_tp_token` slot for easier - superclass identification, which attempts to resolve the `type checking issue - `__ mentioned in :pep:`630`. - (Contributed in :gh:`124153`.) - -* Add :c:func:`PyUnicode_Equal` function to the limited C API: - test if two strings are equal. - (Contributed by Victor Stinner in :gh:`124502`.) - -* Add :c:func:`PyType_Freeze` function to make a type immutable. - (Contributed by Victor Stinner in :gh:`121654`.) - -* Add :c:func:`PyUnstable_Object_EnableDeferredRefcount` for enabling - deferred reference counting, as outlined in :pep:`703`. - * Add :c:func:`PyMonitoring_FireBranchLeftEvent` and :c:func:`PyMonitoring_FireBranchRightEvent` for generating :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` events, respectively. + (Contributed by Mark Shannon in :gh:`122548`.) -* Add :c:func:`Py_fopen` function to open a file. Similar to the - :c:func:`!fopen` function, but the *path* parameter is a Python object and an - exception is set on error. Add also :c:func:`Py_fclose` function to close a - file. - (Contributed by Victor Stinner in :gh:`127350`.) +* Add :c:func:`PyType_Freeze` function to make a type immutable. + (Contributed by Victor Stinner in :gh:`121654`.) + +* Add :c:func:`PyType_GetBaseByToken` and :c:data:`Py_tp_token` slot + for easier superclass identification, which attempts to resolve the + type checking issue mentioned in :pep:`PEP 630 <630#type-checking>`. + (Contributed in :gh:`124153`.) + +* Add a new :c:func:`PyUnicode_Equal` function to test if two + strings are equal. + The function is also added to the Limited C API. + (Contributed by Victor Stinner in :gh:`124502`.) + +* Add a new :c:type:`PyUnicodeWriter` API to create a Python :class:`str` + object, with the following functions: + + * :c:func:`PyUnicodeWriter_Create` + * :c:func:`PyUnicodeWriter_DecodeUTF8Stateful` + * :c:func:`PyUnicodeWriter_Discard` + * :c:func:`PyUnicodeWriter_Finish` + * :c:func:`PyUnicodeWriter_Format` + * :c:func:`PyUnicodeWriter_WriteASCII` + * :c:func:`PyUnicodeWriter_WriteChar` + * :c:func:`PyUnicodeWriter_WriteRepr` + * :c:func:`PyUnicodeWriter_WriteStr` + * :c:func:`PyUnicodeWriter_WriteSubstring` + * :c:func:`PyUnicodeWriter_WriteUCS4` + * :c:func:`PyUnicodeWriter_WriteUTF8` + * :c:func:`PyUnicodeWriter_WriteWideChar` + + (Contributed by Victor Stinner in :gh:`119182`.) * The ``k`` and ``K`` formats in :c:func:`PyArg_ParseTuple` and similar functions now use :meth:`~object.__index__` if available, like all other integer formats. (Contributed by Serhiy Storchaka in :gh:`112068`.) -* Add macros :c:func:`Py_PACK_VERSION` and :c:func:`Py_PACK_FULL_VERSION` for - bit-packing Python version numbers. - (Contributed by Petr Viktorin in :gh:`128629`.) +* Add support for a new ``p`` format unit in :c:func:`Py_BuildValue` + that produces a Python :class:`bool` object from a C integer. + (Contributed by Pablo Galindo in :issue:`45325`.) -* Add :c:func:`PyUnstable_IsImmortal` for determining whether an object is :term:`immortal`, - for debugging purposes. +* Add :c:func:`PyUnstable_IsImmortal` for determining if + an object is :term:`immortal`, for debugging purposes. + (Contributed by Peter Bierma in :gh:`128509`.) -* Add :c:func:`PyImport_ImportModuleAttr` and - :c:func:`PyImport_ImportModuleAttrString` helper functions to import a module - and get an attribute of the module. - (Contributed by Victor Stinner in :gh:`128911`.) +* Add :c:func:`PyUnstable_Object_EnableDeferredRefcount` for enabling + deferred reference counting, as outlined in :pep:`703`. -* Add support for a new ``p`` format unit in :c:func:`Py_BuildValue` that allows - taking a C integer and produces a Python :class:`bool` object. (Contributed by - Pablo Galindo in :issue:`45325`.) - -* Add :c:func:`PyUnstable_Object_IsUniqueReferencedTemporary` to determine if an object - is a unique temporary object on the interpreter's operand stack. This can - be used in some cases as a replacement for checking if :c:func:`Py_REFCNT` - is ``1`` for Python objects passed as arguments to C API functions. - -* Add :c:func:`PyUnstable_Object_IsUniquelyReferenced` as a replacement for - ``Py_REFCNT(op) == 1`` on :term:`free threaded ` builds. +* Add :c:func:`PyUnstable_Object_IsUniquelyReferenced` as + a replacement for ``Py_REFCNT(op) == 1`` on :term:`free threaded + ` builds. (Contributed by Peter Bierma in :gh:`133140`.) +* Add :c:func:`PyUnstable_Object_IsUniqueReferencedTemporary` to + determine if an object is a unique temporary object on the + interpreter's operand stack. + This can be used in some cases as a replacement for checking + if :c:func:`Py_REFCNT` is ``1`` for Python objects passed + as arguments to C API functions. + (Contributed by Sam Gross in :gh:`133164`.) + Limited C API changes --------------------- -* In the limited C API 3.14 and newer, :c:func:`Py_TYPE` and - :c:func:`Py_REFCNT` are now implemented as an opaque function call to hide - implementation details. +* In the limited C API version 3.14 and newer, :c:func:`Py_TYPE` and + :c:func:`Py_REFCNT` are now implemented as an opaque function call + to hide implementation details. (Contributed by Victor Stinner in :gh:`120600` and :gh:`124127`.) * Remove the :c:macro:`PySequence_Fast_GET_SIZE`, - :c:macro:`PySequence_Fast_GET_ITEM` and :c:macro:`PySequence_Fast_ITEMS` - macros from the limited C API, since these macros never worked in the limited - C API. Keep :c:func:`PySequence_Fast` in the limited C API. + :c:macro:`PySequence_Fast_GET_ITEM`, + and :c:macro:`PySequence_Fast_ITEMS` + macros from the limited C API, since they have always been broken + in the limited C API. (Contributed by Victor Stinner in :gh:`91417`.) +.. _whatsnew314-c-api-removed: + +Removed C APIs +-------------- + +* Creating :c:data:`immutable types ` with + mutable bases was deprecated in Python 3.12, + and now raises a :exc:`TypeError`. + (Contributed by Nikita Sobolev in :gh:`119775`.) + +* Remove ``PyDictObject.ma_version_tag`` member, which was deprecated + in Python 3.12. + Use the :c:func:`PyDict_AddWatcher` API instead. + (Contributed by Sam Gross in :gh:`124296`.) + +* Remove the private ``_Py_InitializeMain()`` function. + It was a :term:`provisional API` added to Python 3.8 by :pep:`587`. + (Contributed by Victor Stinner in :gh:`129033`.) + +* Remove the undocumented APIs :c:macro:`!Py_C_RECURSION_LIMIT` + and :c:member:`!PyThreadState.c_recursion_remaining`. + These were added in 3.13 and have been removed without deprecation. + Use :c:func:`Py_EnterRecursiveCall` to guard against runaway + recursion in C code. + (Removed by Petr Viktorin in :gh:`133079`, see also :gh:`130396`.) + + .. _whatsnew314-c-api-deprecated: -Deprecated ----------- +Deprecated C APIs +----------------- -* The :c:macro:`!Py_HUGE_VAL` macro is :term:`soft deprecated`, - use :c:macro:`!Py_INFINITY` instead. +* The :c:macro:`!Py_HUGE_VAL` macro is now :term:`soft deprecated`. + Use :c:macro:`!INFINITY` instead. (Contributed by Sergey B Kirpichev in :gh:`120026`.) -* Macros :c:macro:`!Py_IS_NAN`, :c:macro:`!Py_IS_INFINITY` - and :c:macro:`!Py_IS_FINITE` are :term:`soft deprecated`, - use instead :c:macro:`!isnan`, :c:macro:`!isinf` and - :c:macro:`!isfinite` available from :file:`math.h` - since C99. (Contributed by Sergey B Kirpichev in :gh:`119613`.) +* The :c:macro:`!Py_IS_NAN`, :c:macro:`!Py_IS_INFINITY`, + and :c:macro:`!Py_IS_FINITE` macros are now :term:`soft deprecated`. + Use :c:macro:`!isnan`, :c:macro:`!isinf` and :c:macro:`!isfinite` + instead, available from :file:`math.h` since C99. + (Contributed by Sergey B Kirpichev in :gh:`119613`.) -* Non-tuple sequences are deprecated as argument for the ``(items)`` - format unit in :c:func:`PyArg_ParseTuple` and other - :ref:`argument parsing ` functions if *items* contains - format units which store a :ref:`borrowed buffer ` - or a :term:`borrowed reference`. +* Non-tuple sequences are now deprecated as argument for the ``(items)`` + format unit in :c:func:`PyArg_ParseTuple` and other :ref:`argument + parsing ` functions if *items* contains format units + which store a :ref:`borrowed buffer ` or a + :term:`borrowed reference`. (Contributed by Serhiy Storchaka in :gh:`50333`.) -* The previously undocumented function :c:func:`PySequence_In` is :term:`soft deprecated`. +* The ``_PyMonitoring_FireBranchEvent`` function is now deprecated + and should be replaced with calls to + :c:func:`PyMonitoring_FireBranchLeftEvent` and + :c:func:`PyMonitoring_FireBranchRightEvent`. + +* The previously undocumented function :c:func:`PySequence_In` is + now :term:`soft deprecated`. Use :c:func:`PySequence_Contains` instead. (Contributed by Yuki Kobayashi in :gh:`127896`.) .. Add C API deprecations above alphabetically, not here at the end. -* The ``PyMonitoring_FireBranchEvent`` function is deprecated and should - be replaced with calls to :c:func:`PyMonitoring_FireBranchLeftEvent` - and :c:func:`PyMonitoring_FireBranchRightEvent`. - -* The following private functions are deprecated and planned for removal in - Python 3.18: - - * :c:func:`!_PyBytes_Join`: use :c:func:`PyBytes_Join`. - * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`. - * :c:func:`!_PyDict_Pop()`: use :c:func:`PyDict_Pop`. - * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`. - * :c:func:`!_PyLong_FromDigits` and :c:func:`!_PyLong_New`: - use :c:func:`PyLongWriter_Create`. - * :c:func:`!_PyThreadState_UncheckedGet`: use :c:func:`PyThreadState_GetUnchecked`. - * :c:func:`!_PyUnicode_AsString`: use :c:func:`PyUnicode_AsUTF8`. - * :c:func:`!_PyUnicodeWriter_Init`: - replace ``_PyUnicodeWriter_Init(&writer)`` with - :c:func:`writer = PyUnicodeWriter_Create(0) `. - * :c:func:`!_PyUnicodeWriter_Finish`: - replace ``_PyUnicodeWriter_Finish(&writer)`` with - :c:func:`PyUnicodeWriter_Finish(writer) `. - * :c:func:`!_PyUnicodeWriter_Dealloc`: - replace ``_PyUnicodeWriter_Dealloc(&writer)`` with - :c:func:`PyUnicodeWriter_Discard(writer) `. - * :c:func:`!_PyUnicodeWriter_WriteChar`: - replace ``_PyUnicodeWriter_WriteChar(&writer, ch)`` with - :c:func:`PyUnicodeWriter_WriteChar(writer, ch) `. - * :c:func:`!_PyUnicodeWriter_WriteStr`: - replace ``_PyUnicodeWriter_WriteStr(&writer, str)`` with - :c:func:`PyUnicodeWriter_WriteStr(writer, str) `. - * :c:func:`!_PyUnicodeWriter_WriteSubstring`: - replace ``_PyUnicodeWriter_WriteSubstring(&writer, str, start, end)`` with - :c:func:`PyUnicodeWriter_WriteSubstring(writer, str, start, end) `. - * :c:func:`!_PyUnicodeWriter_WriteASCIIString`: - replace ``_PyUnicodeWriter_WriteASCIIString(&writer, str)`` with - :c:func:`PyUnicodeWriter_WriteASCII(writer, str) `. - * :c:func:`!_PyUnicodeWriter_WriteLatin1String`: - replace ``_PyUnicodeWriter_WriteLatin1String(&writer, str)`` with - :c:func:`PyUnicodeWriter_WriteUTF8(writer, str) `. - * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`. - * :c:func:`!_Py_fopen_obj`: use :c:func:`Py_fopen`. - - The `pythoncapi-compat project`_ can be used to get these new public - functions on Python 3.13 and older. - (Contributed by Victor Stinner in :gh:`128863`.) - .. include:: ../deprecations/c-api-pending-removal-in-3.15.rst .. include:: ../deprecations/c-api-pending-removal-in-3.16.rst @@ -3073,33 +3117,25 @@ Deprecated .. include:: ../deprecations/c-api-pending-removal-in-future.rst -.. _whatsnew314-c-api-removed: +.. _whatsnew314-build-changes: -Removed -------- - -* Creating :c:data:`immutable types ` with mutable - bases was deprecated since 3.12 and now raises a :exc:`TypeError`. - -* Remove ``PyDictObject.ma_version_tag`` member which was deprecated since - Python 3.12. Use the :c:func:`PyDict_AddWatcher` API instead. - (Contributed by Sam Gross in :gh:`124296`.) - -* Remove the private ``_Py_InitializeMain()`` function. It was a - :term:`provisional API` added to Python 3.8 by :pep:`587`. - (Contributed by Victor Stinner in :gh:`129033`.) - -* The undocumented APIs :c:macro:`!Py_C_RECURSION_LIMIT` and - :c:member:`!PyThreadState.c_recursion_remaining`, added in 3.13, are removed - without a deprecation period. - Please use :c:func:`Py_EnterRecursiveCall` to guard against runaway recursion - in C code. - (Removed in :gh:`133079`, see also :gh:`130396`.) - - -Build Changes +Build changes ============= +* :pep:`776`: Emscripten is now an officially supported platform at + :pep:`tier 3 <11#tier-3>`. As a part of this effort, more than 25 bugs in + `Emscripten libc`__ were fixed. Emscripten now includes support + for :mod:`ctypes`, :mod:`termios`, and :mod:`fcntl`, as well as + experimental support for the new :ref:`default interactive shell + `. + (Contributed by R. Hood Chatham in :gh:`127146`, :gh:`127683`, and :gh:`136931`.) + + __ https://emscripten.org/docs/porting/emscripten-runtime-environment.html + +* Official Android binary releases are now provided on python.org__. + + __ https://www.python.org/downloads/android/ + * GNU Autoconf 2.72 is now required to generate :file:`configure`. (Contributed by Erlend Aasland in :gh:`115765`.) @@ -3121,12 +3157,22 @@ Build Changes * The new :file:`configure` option :option:`--with-tail-call-interp` may be used to enable the experimental tail call interpreter. - See :ref:`whatsnew314-tail-call` for further details. + See :ref:`whatsnew314-tail-call-interpreter` for further details. * To disable the new remote debugging support, use the :option:`--without-remote-debug` :file:`configure` option. This may be useful for security reasons. +* iOS and macOS apps can now be configured to redirect ``stdout`` and + ``stderr`` content to the system log. + (Contributed by Russell Keith-Magee in :gh:`127592`.) + +* The iOS testbed is now able to stream test output while the test is running. + The testbed can also be used to run the test suite of projects other than + CPython itself. + (Contributed by Russell Keith-Magee in :gh:`127592`.) + + .. _whatsnew314-build_details: :file:`build-details.json` @@ -3147,6 +3193,7 @@ which can be found by running ``sysconfig.get_path('stdlib')``. :pep:`739` -- ``build-details.json`` 1.0 -- a static description file for Python build details + .. _whatsnew314-no-more-pgp: Discontinuation of PGP signatures @@ -3163,6 +3210,66 @@ This change in release process was specified in :pep:`761`. .. _Sigstore: https://www.sigstore.dev/ +.. _whatsnew314-free-threaded-now-supported: + +Free-threaded Python is officially supported +-------------------------------------------- + +The free-threaded build of Python is now supported and no longer experimental. +This is the start of `phase II `__ where +free-threaded Python is officially supported but still optional. + +The free-threading team are confident that the project is on the right path, +and appreciate the continued dedication from everyone working to make +free-threading ready for broader adoption across the Python community. + +With these recommendations and the acceptance of this PEP, the Python developer +community should broadly advertise that free-threading is a supported +Python build option now and into the future, and that it will not be removed +without a proper deprecation schedule. + +Any decision to transition to `phase III `__, +with free-threading as the default or sole build of Python is still undecided, +and dependent on many factors both within CPython itself and the community. +This decision is for the future. + +.. seealso:: + + :pep:`779` + + `PEP 779's acceptance `__ + + +.. _whatsnew314-jit-compiler: + +Binary releases for the experimental just-in-time compiler +---------------------------------------------------------- + +The official macOS and Windows release binaries now include an *experimental* +just-in-time (JIT) compiler. Although it is **not** recommended for production +use, it can be tested by setting :envvar:`PYTHON_JIT=1 ` as an +environment variable. Downstream source builds and redistributors can use the +:option:`--enable-experimental-jit=yes-off` configuration option for similar +behavior. + +The JIT is at an early stage and still in active development. As such, the +typical performance impact of enabling it can range from 10% slower to 20% +faster, depending on workload. To aid in testing and evaluation, a set of +introspection functions has been provided in the :data:`sys._jit` namespace. +:func:`sys._jit.is_available` can be used to determine if the current executable +supports JIT compilation, while :func:`sys._jit.is_enabled` can be used to tell +if JIT compilation has been enabled for the current process. + +Currently, the most significant missing functionality is that native debuggers +and profilers like ``gdb`` and ``perf`` are unable to unwind through JIT frames +(Python debuggers and profilers, like :mod:`pdb` or :mod:`profile`, continue to +work without modification). Free-threaded builds do not support JIT compilation. + +Please report any bugs or major performance regressions that you encounter! + +.. seealso:: :pep:`744` + + Porting to Python 3.14 ====================== @@ -3173,6 +3280,20 @@ that may require changes to your code. Changes in the Python API ------------------------- +* On Unix platforms other than macOS, *forkserver* is now the default + :ref:`start method ` for :mod:`multiprocessing` + and :class:`~concurrent.futures.ProcessPoolExecutor`, instead of *fork*. + + See :ref:`(1) ` and + :ref:`(2) ` for details. + + If you encounter :exc:`NameError`\s or pickling errors coming out of + :mod:`multiprocessing` or :mod:`concurrent.futures`, see the + :ref:`forkserver restrictions `. + + This change does not affect Windows or macOS, where :ref:`'spawn' + ` remains the default start method. + * :class:`functools.partial` is now a method descriptor. Wrap it in :func:`staticmethod` if you want to preserve the old behavior. (Contributed by Serhiy Storchaka and Dominykas Grigonis in :gh:`121027`.) @@ -3195,10 +3316,106 @@ Changes in the Python API (Contributed by Jelle Zijlstra in :gh:`105499`.) * The runtime behavior of annotations has changed in various ways; see - :ref:`above ` for details. While most code that interacts + :ref:`above ` for details. While most code that interacts with annotations should continue to work, some undocumented details may behave differently. +* As part of making the :mod:`mimetypes` CLI public, + it now exits with ``1`` on failure instead of ``0`` + and ``2`` on incorrect command-line parameters instead of ``1``. + Error messages are now printed to stderr. + +* The ``\B`` pattern in regular expression now matches the empty string + when given as the entire pattern, which may cause behavioural changes. + +* On FreeBSD, :data:`sys.platform` no longer contains the major version number. + + +.. _whatsnew314-porting-annotations: + +Changes in annotations (:pep:`649` and :pep:`749`) +-------------------------------------------------- + +This section contains guidance on changes that may be needed to annotations +or Python code that interacts with or introspects annotations, +due to the changes related to :ref:`deferred evaluation of annotations +`. + +In the majority of cases, working code from older versions of Python +will not require any changes. + + +Implications for annotated code +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you define annotations in your code (for example, for use with a static type +checker), then this change probably does not affect you: you can keep +writing annotations the same way you did with previous versions of Python. + +You will likely be able to remove quoted strings in annotations, which are frequently +used for forward references. Similarly, if you use ``from __future__ import annotations`` +to avoid having to write strings in annotations, you may well be able to +remove that import once you support only Python 3.14 and newer. +However, if you rely on third-party libraries that read annotations, +those libraries may need changes to support unquoted annotations before they +work as expected. + + +Implications for readers of ``__annotations__`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If your code reads the :attr:`~object.__annotations__` attribute on objects, +you may want to make changes in order to support code that relies on +deferred evaluation of annotations. +For example, you may want to use :func:`annotationlib.get_annotations` with +the :attr:`~annotationlib.Format.FORWARDREF` format, +as the :mod:`dataclasses` module now does. + +The external :pypi:`typing_extensions` package provides partial backports +of some of the functionality of the :mod:`annotationlib` module, +such as the :class:`~annotationlib.Format` enum and +the :func:`~annotationlib.get_annotations` function. +These can be used to write cross-version code that takes advantage of +the new behavior in Python 3.14. + + +Related changes +^^^^^^^^^^^^^^^ + +The changes in Python 3.14 are designed to rework how :attr:`!__annotations__` +works at runtime while minimizing breakage to code that contains +annotations in source code and to code that reads :attr:`!__annotations__`. +However, if you rely on undocumented details of the annotation behavior +or on private functions in the standard library, there are many ways in which +your code may not work in Python 3.14. +To safeguard your code against future changes, only use the documented +functionality of the :mod:`annotationlib` module. + +In particular, do not read annotations directly from the namespace dictionary +attribute of type objects. +Use :func:`annotationlib.get_annotate_from_class_namespace` during class +construction and :func:`annotationlib.get_annotations` afterwards. + +In previous releases, it was sometimes possible to access class annotations +from an instance of an annotated class. This behavior was undocumented +and accidental, and will no longer work in Python 3.14. + + +``from __future__ import annotations`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In Python 3.7, :pep:`563` introduced the ``from __future__ import annotations`` +:ref:`future statement `, which turns all annotations into strings. + +However, this statement is now deprecated and it is expected to be removed +in a future version of Python. +This removal will not happen until after Python 3.13 reaches its end of life +in 2029, being the last version of Python without support for deferred +evaluation of annotations. + +In Python 3.14, the behavior of code using ``from __future__ import annotations`` +is unchanged. + Changes in the C API -------------------- @@ -3255,3 +3472,17 @@ Changes in the C API functions on Python 3.13 and older. .. _pythoncapi-compat project: https://github.com/python/pythoncapi-compat/ + + +Notable changes in 3.14.5 +========================= + +gc +-- + +* The incremental garbage collector shipped in Python 3.14.0-3.14.4 has been + reverted back to the generational garbage collector from 3.13, + due to a number of `reports + `__ + of significant memory pressure in production environments. + See :ref:`whatsnew314-incremental-gc` for details. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 56ef80c0686..78e464f2a5a 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -3,7 +3,7 @@ What's new in Python 3.15 **************************** -:Editor: TBD +:Editor: Hugo van Kemenade .. Rules for maintenance: @@ -56,8 +56,8 @@ For full details, see the :ref:`changelog `. so it's worth checking back even after reading earlier versions. -Summary --- release highlights -============================== +Summary -- Release highlights +============================= .. This section singles out the most important changes in Python 3.15. Brevity is key. @@ -65,109 +65,390 @@ Summary --- release highlights .. PEP-sized items next. - +* :pep:`810`: :ref:`Explicit lazy imports for faster startup times + ` +* :pep:`814`: :ref:`Add frozendict built-in type + ` +* :pep:`661`: :ref:`Add sentinel built-in type + ` +* :pep:`799`: :ref:`A dedicated profiling package for organizing Python + profiling tools ` +* :pep:`799`: :ref:`Tachyon: High frequency statistical sampling profiler + ` +* :pep:`798`: :ref:`Unpacking in comprehensions + ` +* :pep:`686`: :ref:`Python now uses UTF-8 as the default encoding + ` +* :pep:`728`: ``TypedDict`` with typed extra items +* :pep:`747`: :ref:`Annotating type forms with TypeForm + ` +* :pep:`800`: Disjoint bases in the type system +* :pep:`782`: :ref:`A new PyBytesWriter C API to create a Python bytes object + ` +* :pep:`803`: :ref:`Stable ABI for Free-Threaded Builds ` +* :pep:`831`: :ref:`Frame pointers everywhere ` +* :ref:`The JIT compiler has been significantly upgraded ` +* :ref:`Improved error messages ` +* :ref:`The official Windows 64-bit binaries now use the tail-calling interpreter + ` New features ============ +.. _whatsnew315-lazy-imports: + +:pep:`810`: Explicit lazy imports +--------------------------------- + +Large Python applications often suffer from slow startup times. A +significant contributor to this problem is the import system: when a module +is imported, Python must locate the file, read it from disk, compile it to +bytecode, and execute all top-level code. For applications with deep +dependency trees, this process can take seconds, even when most of the +imported code is never actually used during a particular run. + +Developers have worked around this by moving imports inside functions, using +:mod:`importlib` to load modules on demand, or restructuring code to avoid +unnecessary dependencies. These approaches work but make code harder to read +and maintain, scatter import statements throughout the codebase, and require +discipline to apply consistently. + +Python now provides a cleaner solution through explicit :keyword:`lazy` +imports using the new ``lazy`` soft keyword. When you mark an import as +lazy, Python defers the actual module loading until the imported name is +first used. This gives you the organizational benefits of declaring all +imports at the top of the file while only paying the loading cost for +modules you actually use. + +The ``lazy`` keyword works with both ``import`` and ``from ... import`` +statements. When you write ``lazy import heavy_module``, Python does not +immediately load the module. Instead, it creates a lightweight proxy object. +The actual module loading happens transparently when you first access the +name: + +.. code-block:: python + + lazy import json + lazy from pathlib import Path + + print("Starting up...") # json and pathlib not loaded yet + + data = json.loads('{"key": "value"}') # json loads here + p = Path(".") # pathlib loads here + +This mechanism is particularly useful for applications that import many +modules at the top level but may only use a subset of them in any given run. +The deferred loading reduces startup latency without requiring code +restructuring or conditional imports scattered throughout the codebase. + +In the case where loading a lazily imported module fails (for example, if +the module does not exist), Python raises the exception at the point of +first use rather than at import time. The associated traceback includes both +the location where the name was accessed and the original import statement, +making it straightforward to diagnose and debug the failure. + +For cases where you want to enable lazy loading globally without modifying +source code, Python provides the :option:`-X lazy_imports <-X>` command-line +option and the :envvar:`PYTHON_LAZY_IMPORTS` environment variable. Both +accept three values: ``all`` makes all imports lazy by default, ``none`` +disables lazy imports entirely (even explicit ``lazy`` statements become +eager), and ``normal`` (the default) respects the ``lazy`` keyword in source +code. The :func:`sys.set_lazy_imports` and :func:`sys.get_lazy_imports` +functions allow changing and querying this mode at runtime. + +For more selective control, :func:`sys.set_lazy_imports_filter` accepts a +callable that determines whether a specific module should be loaded lazily. +The filter receives three arguments: the importing module's name (or +``None``), the imported module's name, and the fromlist (or ``None`` for +regular imports). It should return ``True`` to allow the import to be lazy, +or ``False`` to force eager loading. This allows patterns like making only +your own application's modules lazy while keeping third-party dependencies +eager: + +.. code-block:: python + + import sys + + def myapp_filter(importing, imported, fromlist): + return imported.startswith("myapp.") + sys.set_lazy_imports_filter(myapp_filter) + sys.set_lazy_imports("all") + + import myapp.slow_module # lazy (matches filter) + import json # eager (does not match filter) + +The proxy type itself is available as :data:`types.LazyImportType` for code +that needs to detect lazy imports programmatically. + +There are some restrictions on where the ``lazy`` keyword can be used. Lazy +imports are only permitted at module scope; using ``lazy`` inside a +function, class body, or ``try``/``except``/``finally`` block raises a +:exc:`SyntaxError`. Neither star imports nor future imports can be lazy +(``lazy from module import *`` and ``lazy from __future__ import ...`` both +raise :exc:`SyntaxError`). + +For code that cannot use the ``lazy`` keyword directly (for example, when +supporting Python versions older than 3.15 while still using lazy +imports on 3.15+), a module can define +:attr:`~module.__lazy_modules__` as a container of fully qualified module +name strings. Regular ``import`` statements for those modules are then treated +as lazy, with the same semantics as the ``lazy`` keyword:: + + __lazy_modules__ = ["json", "pathlib"] + + import json # lazy + import os # still eager + +.. seealso:: :pep:`810` for the full specification and rationale. + +(Contributed by Pablo Galindo Salgado and Dino Viehland in :gh:`142349`.) + + +.. _whatsnew315-frozendict: + +:pep:`814`: Add frozendict built-in type +---------------------------------------- + +A new :term:`immutable` type, :class:`frozendict`, is added to the :mod:`builtins` module. +It does not allow modification after creation. A :class:`!frozendict` is not a subclass of ``dict``; +it inherits directly from ``object``. A :class:`!frozendict` is :term:`hashable` +as long as all of its keys and values are hashable. A :class:`!frozendict` preserves +insertion order, but comparison does not take order into account. + +For example:: + + >>> a = frozendict(x=1, y=2) + >>> a + frozendict({'x': 1, 'y': 2}) + >>> a['z'] = 3 + Traceback (most recent call last): + File "", line 1, in + a['z'] = 3 + ~^^^^^ + TypeError: 'frozendict' object does not support item assignment + >>> b = frozendict(y=2, x=1) + >>> hash(a) == hash(b) + True + >>> a == b + True + +The following standard library modules have been updated to accept +:class:`!frozendict`: :mod:`copy`, :mod:`decimal`, :mod:`json`, :mod:`marshal`, +:mod:`plistlib` (only for serialization), :mod:`pickle`, :mod:`pprint` and +:mod:`xml.etree.ElementTree`. + +:func:`eval` and :func:`exec` accept :class:`!frozendict` for *globals*, and +:func:`type` and :meth:`str.maketrans` accept :class:`!frozendict` for *dict*. + +Code checking for :class:`dict` type using ``isinstance(arg, dict)`` can be +updated to ``isinstance(arg, (dict, frozendict))`` to accept also the +:class:`!frozendict` type, or to ``isinstance(arg, collections.abc.Mapping)`` +to accept also other mapping types such as :class:`~types.MappingProxyType`. + +.. seealso:: :pep:`814` for the full specification and rationale. + +(Contributed by Victor Stinner and Donghee Na in :gh:`141510`.) + + +.. _whatsnew315-sentinel: + +:pep:`661`: Add sentinel built-in type +-------------------------------------- + +A new :class:`sentinel` type is added to the :mod:`builtins` module for +creating unique sentinel values with a concise representation. Sentinel +objects preserve identity when copied, support use in type expressions with +the ``|`` operator, and can be pickled when they are importable by module and +name. + +(PEP by Tal Einat; contributed by Jelle Zijlstra in :gh:`148829`.) + + +.. _whatsnew315-profiling-package: + +:pep:`799`: A dedicated profiling package +----------------------------------------- + +A new :mod:`profiling` module has been added to organize Python's built-in +profiling tools under a single, coherent namespace. This module contains: + +* :mod:`profiling.tracing`: deterministic function-call tracing (relocated from + ``cProfile``). +* :mod:`profiling.sampling`: a new statistical sampling profiler (named Tachyon). + +The ``cProfile`` module remains as an alias for backwards compatibility. +The :mod:`profile` module is deprecated and will be removed in Python 3.17. + +.. seealso:: :pep:`799` for further details. + +(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`138122`.) + + .. _whatsnew315-sampling-profiler: -High frequency statistical sampling profiler --------------------------------------------- +Tachyon: High frequency statistical sampling profiler +----------------------------------------------------- -A new statistical sampling profiler has been added to the new :mod:`!profiling` module as -:mod:`!profiling.sampling`. This profiler enables low-overhead performance analysis of +.. image:: ../../Lib/profiling/sampling/_assets/tachyon-logo.png + :alt: Tachyon profiler logo + :align: center + :width: 200px + +A new statistical sampling profiler (Tachyon) has been added as +:mod:`profiling.sampling`. This profiler enables low-overhead performance analysis of running Python processes without requiring code modification or process restart. -Unlike deterministic profilers (:mod:`cProfile` and :mod:`profile`) that instrument +Unlike deterministic profilers (such as :mod:`profiling.tracing`) that instrument every function call, the sampling profiler periodically captures stack traces from running processes. This approach provides virtually zero overhead while achieving sampling rates of **up to 1,000,000 Hz**, making it the fastest sampling profiler available for Python (at the time of its contribution) and ideal for debugging -performance issues in production environments. +performance issues in production environments. This capability is particularly +valuable for debugging performance issues in production systems where traditional +profiling approaches would be too intrusive. Key features include: * **Zero-overhead profiling**: Attach to any running Python process without - affecting its performance -* **No code modification required**: Profile existing applications without restart -* **Real-time statistics**: Monitor sampling quality during data collection -* **Multiple output formats**: Generate both detailed statistics and flamegraph data -* **Thread-aware profiling**: Option to profile all threads or just the main thread + affecting its performance. Ideal for production debugging where you can't afford + to restart or slow down your application. -Profile process 1234 for 10 seconds with default settings:: +* **No code modification required**: Profile existing applications without restart. + Simply point the profiler at a running process by PID and start collecting data. - python -m profiling.sampling 1234 +* **Flexible target modes**: -Profile with custom interval and duration, save to file:: + * Profile running processes by PID (``attach``) - attach to already-running applications + * Run and profile scripts directly (``run``) - profile from the very start of execution + * Execute and profile modules (``run -m``) - profile packages run as ``python -m module`` - python -m profiling.sampling -i 50 -d 30 -o profile.stats 1234 +* **Multiple profiling modes**: Choose what to measure based on your performance investigation: -Generate collapsed stacks for flamegraph:: + * **Wall-clock time** (``--mode wall``, default): Measures real elapsed time including I/O, + network waits, and blocking operations. Use this to understand where your program spends + calendar time, including when waiting for external resources. + * **CPU time** (``--mode cpu``): Measures only active CPU execution time, excluding I/O waits + and blocking. Use this to identify CPU-bound bottlenecks and optimize computational work. + * **GIL-holding time** (``--mode gil``): Measures time spent holding Python's Global Interpreter + Lock. Use this to identify which threads dominate GIL usage in multi-threaded applications. + * **Exception handling time** (``--mode exception``): Captures samples only from threads with + an active exception. Use this to analyze exception handling overhead. - python -m profiling.sampling --collapsed 1234 +* **Thread-aware profiling**: Option to profile all threads (``-a``) or just the main thread, + essential for understanding multi-threaded application behavior. -Profile all threads and sort by total time:: +* **Multiple output formats**: Choose the visualization that best fits your workflow: - python -m profiling.sampling -a --sort-tottime 1234 + * ``--pstats``: Detailed tabular statistics compatible with :mod:`pstats`. Shows function-level + timing with direct and cumulative samples. Best for detailed analysis and integration with + existing Python profiling tools. + * ``--collapsed``: Generates collapsed stack traces (one line per stack). This format is + specifically designed for creating flame graphs with external tools like Brendan Gregg's + FlameGraph scripts or speedscope. + * ``--flamegraph``: Generates a self-contained interactive HTML flame graph using D3.js. + Opens directly in your browser for immediate visual analysis. Flame graphs show the call + hierarchy where width represents time spent, making it easy to spot bottlenecks at a glance. + * ``--gecko``: Generates Gecko Profiler format compatible with `Firefox Profiler + `__. Upload the output to Firefox Profiler for advanced + timeline-based analysis with features like stack charts, markers, and network activity. + * ``--heatmap``: Generates an interactive HTML heatmap visualization with line-level sample + counts. Creates a directory with per-file heatmaps showing exactly where time is spent + at the source code level. -The profiler generates statistical estimates of where time is spent:: +* **Live interactive mode**: Real-time TUI profiler with a top-like interface (``--live``). + Monitor performance as your application runs with interactive sorting and filtering. - Real-time sampling stats: Mean: 100261.5Hz (9.97µs) Min: 86333.4Hz (11.58µs) Max: 118807.2Hz (8.42µs) Samples: 400001 - Captured 498841 samples in 5.00 seconds - Sample rate: 99768.04 samples/sec - Error rate: 0.72% - Profile Stats: - nsamples sample% tottime (s) cumul% cumtime (s) filename:lineno(function) - 43/418858 0.0 0.000 87.9 4.189 case.py:667(TestCase.run) - 3293/418812 0.7 0.033 87.9 4.188 case.py:613(TestCase._callTestMethod) - 158562/158562 33.3 1.586 33.3 1.586 test_compile.py:725(TestSpecifics.test_compiler_recursion_limit..check_limit) - 129553/129553 27.2 1.296 27.2 1.296 ast.py:46(parse) - 0/128129 0.0 0.000 26.9 1.281 test_ast.py:884(AST_Tests.test_ast_recursion_limit..check_limit) - 7/67446 0.0 0.000 14.2 0.674 test_compile.py:729(TestSpecifics.test_compiler_recursion_limit) - 6/60380 0.0 0.000 12.7 0.604 test_ast.py:888(AST_Tests.test_ast_recursion_limit) - 3/50020 0.0 0.000 10.5 0.500 test_compile.py:727(TestSpecifics.test_compiler_recursion_limit) - 1/38011 0.0 0.000 8.0 0.380 test_ast.py:886(AST_Tests.test_ast_recursion_limit) - 1/25076 0.0 0.000 5.3 0.251 test_compile.py:728(TestSpecifics.test_compiler_recursion_limit) - 22361/22362 4.7 0.224 4.7 0.224 test_compile.py:1368(TestSpecifics.test_big_dict_literal) - 4/18008 0.0 0.000 3.8 0.180 test_ast.py:889(AST_Tests.test_ast_recursion_limit) - 11/17696 0.0 0.000 3.7 0.177 subprocess.py:1038(Popen.__init__) - 16968/16968 3.6 0.170 3.6 0.170 subprocess.py:1900(Popen._execute_child) - 2/16941 0.0 0.000 3.6 0.169 test_compile.py:730(TestSpecifics.test_compiler_recursion_limit) +* **Async-aware profiling**: Profile async/await code with task-based stack reconstruction + (``--async-aware``). See which coroutines are consuming time, with options to show only + running tasks or all tasks including those waiting. - Legend: - nsamples: Direct/Cumulative samples (direct executing / on call stack) - sample%: Percentage of total samples this function was directly executing - tottime: Estimated total time spent directly in this function - cumul%: Percentage of total samples when this function was on the call stack - cumtime: Estimated cumulative time (including time in called functions) - filename:lineno(function): Function location and name +* **Opcode-level profiling**: Gather bytecode opcode information for instruction-level + profiling (``--opcodes``). Shows which bytecode instructions are executing, including + specializations from the adaptive interpreter. - Summary of Interesting Functions: +See :mod:`profiling.sampling` for the complete documentation, including all +available output formats, profiling modes, and configuration options. - Functions with Highest Direct/Cumulative Ratio (Hot Spots): - 1.000 direct/cumulative ratio, 33.3% direct samples: test_compile.py:(TestSpecifics.test_compiler_recursion_limit..check_limit) - 1.000 direct/cumulative ratio, 27.2% direct samples: ast.py:(parse) - 1.000 direct/cumulative ratio, 3.6% direct samples: subprocess.py:(Popen._execute_child) +(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`135953` and :gh:`138122`.) - Functions with Highest Call Frequency (Indirect Calls): - 418815 indirect calls, 87.9% total stack presence: case.py:(TestCase.run) - 415519 indirect calls, 87.9% total stack presence: case.py:(TestCase._callTestMethod) - 159470 indirect calls, 33.5% total stack presence: test_compile.py:(TestSpecifics.test_compiler_recursion_limit) - Functions with Highest Call Magnification (Cumulative/Direct): - 12267.9x call magnification, 159470 indirect calls from 13 direct: test_compile.py:(TestSpecifics.test_compiler_recursion_limit) - 10581.7x call magnification, 116388 indirect calls from 11 direct: test_ast.py:(AST_Tests.test_ast_recursion_limit) - 9740.9x call magnification, 418815 indirect calls from 43 direct: case.py:(TestCase.run) +.. _whatsnew315-unpacking-in-comprehensions: -The profiler automatically identifies performance bottlenecks through statistical -analysis, highlighting functions with high CPU usage and call frequency patterns. +:pep:`798`: Unpacking in Comprehensions +--------------------------------------- -This capability is particularly valuable for debugging performance issues in -production systems where traditional profiling approaches would be too intrusive. +List, set, and dictionary comprehensions, as well as generator expressions, now +support unpacking with ``*`` and ``**``. This extends the unpacking syntax +from :pep:`448` to comprehensions, providing a new syntax for combining an +arbitrary number of iterables or dictionaries into a single flat structure. +This new syntax is a direct alternative to nested comprehensions, +:func:`itertools.chain`, and :meth:`itertools.chain.from_iterable`. For +example:: -(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`135953`.) + >>> lists = [[1, 2], [3, 4], [5]] + >>> [*L for L in lists] # equivalent to [x for L in lists for x in L] + [1, 2, 3, 4, 5] + >>> sets = [{1, 2}, {2, 3}, {3, 4}] + >>> {*s for s in sets} # equivalent to {x for s in sets for x in s} + {1, 2, 3, 4} + + >>> dicts = [{'a': 1}, {'b': 2}, {'a': 3}] + >>> {**d for d in dicts} # equivalent to {k: v for d in dicts for k,v in d.items()} + {'a': 3, 'b': 2} + +Generator expressions can similarly use unpacking to yield values from multiple +iterables:: + + >>> gen = (*L for L in lists) # equivalent to (x for L in lists for x in L) + >>> list(gen) + [1, 2, 3, 4, 5] + +This change also extends to asynchronous generator expressions, such that, for +example, ``(*a async for a in agen())`` is equivalent to ``(x async for a in +agen() for x in a)``. + +.. seealso:: :pep:`798` for further details. + +(Contributed by Adam Hartz in :gh:`143055`.) + +.. _whatsnew315-abi3t: + +:pep:`803`: ``abi3t`` -- Stable ABI for Free-Threaded Builds +------------------------------------------------------------ + +C extensions that target the :ref:`Stable ABI ` can now be +compiled for the new *Stable ABI for Free-Threaded Builds* (also known +as ``abi3t``), which makes them compatible with +:term:`free-threaded builds ` of CPython. +This usually requires some non-trivial changes to the source code; +specifically: + +- Switching to API introduced in :pep:`697` (Python 3.12), such as + negative :c:member:`~PyType_Spec.basicsize` and + :c:func:`PyObject_GetTypeData`, rather than making :c:type:`PyObject` + part of the instance struct; and +- Switching from a ``PyInit_`` function to a new export hook, + :c:func:`PyModExport_* `, introduced for this + purpose in :pep:`793`. + +Note that Stable ABI does not offer all the functionality that CPython +has to offer. +Extensions that cannot switch to ``abi3t`` should continue to build for +the existing Stable ABI (``abi3``) and the version-specific ABI for +free-threading (``cp315t``) separately. + +Stable ABI for Free-Threaded Builds should typically +be selected in a build tool (such as, for example, Setuptools, meson-python, +scikit-build-core, or Maturin). +At the time of writing, these tools did **not** support ``abi3t``. +If this is the case for your tool, compile for ``cp315t`` separately. +If not using a build tool -- or when writing such a tool -- you can select +``abi3t`` by setting the macro :c:macro:`!Py_TARGET_ABI3T` as discussed +in :ref:`abi3-compiling`. + + +.. _whatsnew315-improved-error-messages: Improved error messages ----------------------- @@ -199,45 +480,13 @@ Improved error messages Running this code now produces a clearer suggestion: - .. code-block:: pycon + .. code-block:: pytb Traceback (most recent call last): - File "/home/pablogsal/github/python/main/lel.py", line 42, in - print(container.area) - ^^^^^^^^^^^^^^ - AttributeError: 'Container' object has no attribute 'area'. Did you mean: 'inner.area'? - - -Other language changes -====================== - -* Python now uses UTF-8_ as the default encoding, independent of the system's - environment. This means that I/O operations without an explicit encoding, - e.g. ``open('flying-circus.txt')``, will use UTF-8. - UTF-8 is a widely-supported Unicode_ character encoding that has become a - *de facto* standard for representing text, including nearly every webpage - on the internet, many common file formats, programming languages, and more. - - This only applies when no ``encoding`` argument is given. For best - compatibility between versions of Python, ensure that an explicit ``encoding`` - argument is always provided. The :ref:`opt-in encoding warning ` - can be used to identify code that may be affected by this change. - The special ``encoding='locale'`` argument uses the current locale - encoding, and has been supported since Python 3.10. - - To retain the previous behaviour, Python's UTF-8 mode may be disabled with - the :envvar:`PYTHONUTF8=0 ` environment variable or the - :option:`-X utf8=0 <-X>` command line option. - - .. seealso:: :pep:`686` for further details. - - .. _UTF-8: https://en.wikipedia.org/wiki/UTF-8 - .. _Unicode: https://home.unicode.org/ - - (Contributed by Adam Turner in :gh:`133711`; PEP 686 written by Inada Naoki.) - -* Several error messages incorrectly using the term "argument" have been corrected. - (Contributed by Stan Ulbrych in :gh:`133382`.) + File "/home/pablogsal/github/python/main/lel.py", line 42, in + print(container.area) + ^^^^^^^^^^^^^^ + AttributeError: 'Container' object has no attribute 'area'. Did you mean '.inner.area' instead of '.area'? * The interpreter now tries to provide a suggestion when :func:`delattr` fails due to a missing attribute. @@ -258,6 +507,40 @@ Other language changes (Contributed by Nikita Sobolev and Pranjal Prajapati in :gh:`136588`.) +* Several error messages incorrectly using the term "argument" have been corrected. + (Contributed by Stan Ulbrych in :gh:`133382`.) + + +Other language changes +====================== + +.. _whatsnew315-utf8-default: + +* Python now uses UTF-8_ as the default encoding, independent of the system's + environment. This means that I/O operations without an explicit encoding, + for example, ``open('flying-circus.txt')``, will use UTF-8. + UTF-8 is a widely-supported Unicode_ character encoding that has become a + *de facto* standard for representing text, including nearly every webpage + on the internet, many common file formats, programming languages, and more. + + This only applies when no ``encoding`` argument is given. For best + compatibility between versions of Python, ensure that an explicit ``encoding`` + argument is always provided. The :ref:`opt-in encoding warning ` + can be used to identify code that may be affected by this change. + The special ``encoding='locale'`` argument uses the current locale + encoding, and has been supported since Python 3.10. + + To retain the previous behaviour, Python's UTF-8 mode may be disabled with + the :envvar:`PYTHONUTF8=0 ` environment variable or the + :option:`-X utf8=0 <-X>` command-line option. + + .. seealso:: :pep:`686` for further details. + + .. _UTF-8: https://en.wikipedia.org/wiki/UTF-8 + .. _Unicode: https://home.unicode.org/ + + (Contributed by Adam Turner in :gh:`133711`; PEP 686 written by Inada Naoki.) + * Unraisable exceptions are now highlighted with color by default. This can be controlled by :ref:`environment variables `. (Contributed by Peter Bierma in :gh:`134170`.) @@ -273,16 +556,332 @@ Other language changes This speeds up class creation, and helps avoid reference cycles. (Contributed by Petr Viktorin in :gh:`135228`.) +* The :option:`-W` option and the :envvar:`PYTHONWARNINGS` environment variable + can now specify regular expressions instead of literal strings to match + the warning message and the module name, if the corresponding field starts + and ends with a forward slash (``/``). + (Contributed by Serhiy Storchaka in :gh:`134716`.) + +* Functions that take timestamp or timeout arguments now accept any real + numbers (such as :class:`~decimal.Decimal` and :class:`~fractions.Fraction`), + not only integers or floats, although this does not improve precision. + (Contributed by Serhiy Storchaka in :gh:`67795`.) + +.. _whatsnew315-bytearray-take-bytes: + +* Added :meth:`bytearray.take_bytes(n=None, /) ` to take + bytes out of a :class:`bytearray` without copying. This enables optimizing code + which must return :class:`bytes` after working with a mutable buffer of bytes + such as data buffering, network protocol parsing, encoding, decoding, + and compression. Common code patterns which can be optimized with + :func:`~bytearray.take_bytes` are listed below. + + .. list-table:: Suggested optimizing refactors + :header-rows: 1 + + * - Description + - Old + - New + + * - Return :class:`bytes` after working with :class:`bytearray` + - .. code:: python + + def read() -> bytes: + buffer = bytearray(1024) + ... + return bytes(buffer) + + - .. code:: python + + def read() -> bytes: + buffer = bytearray(1024) + ... + return buffer.take_bytes() + + * - Empty a buffer getting the bytes + - .. code:: python + + buffer = bytearray(1024) + ... + data = bytes(buffer) + buffer.clear() + + - .. code:: python + + buffer = bytearray(1024) + ... + data = buffer.take_bytes() + + * - Split a buffer at a specific separator + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + data = bytes(buffer[:n + 1]) + del buffer[:n + 1] + assert data == b'abc' + assert buffer == bytearray(b'def') + + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + data = buffer.take_bytes(n + 1) + + * - Split a buffer at a specific separator; discard after the separator + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + data = bytes(buffer[:n]) + buffer.clear() + assert data == b'abc' + assert len(buffer) == 0 + + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + buffer.resize(n) + data = buffer.take_bytes() + + (Contributed by Cody Maloney in :gh:`139871`.) + +* Many functions related to compiling or parsing Python code, such as + :func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, + and :func:`importlib.abc.InspectLoader.source_to_code`, now allow the module + name to be passed. It is needed to unambiguously :ref:`filter ` + syntax warnings by module name. + (Contributed by Serhiy Storchaka in :gh:`135801`.) + +* Allowed defining the *__dict__* and *__weakref__* :ref:`__slots__ ` + for any class. + (Contributed by Serhiy Storchaka in :gh:`41779`.) + +* Allowed defining any :ref:`__slots__ ` for a class derived from + :class:`tuple` (including classes created by :func:`collections.namedtuple`). + (Contributed by Serhiy Storchaka in :gh:`41779`.) + +* The :class:`slice` type now supports subscription, + making it a :term:`generic type`. + (Contributed by James Hilton-Balfe in :gh:`128335`.) + +* The class :class:`memoryview` now supports the :c:expr:`float complex` and + :c:expr:`double complex` C types: formatting characters ``'F'`` and ``'D'`` + respectively. + (Contributed by Sergey B Kirpichev in :gh:`146151`.) + +* Allow the *count* argument of :meth:`bytes.replace` to be a keyword. + (Contributed by Stan Ulbrych in :gh:`147856`.) + +* Unary plus is now accepted in :keyword:`match` literal patterns, mirroring + the existing support for unary minus. + (Contributed by Bartosz Sławecki in :gh:`145239`.) + +* The import system now acquires per-module locks in hierarchical order + (parent packages before their submodules). This fixes a long-standing + deadlock where one thread importing ``pkg.sub`` and another importing + ``pkg.sub.mod`` could each block the other when ``pkg/sub/__init__.py`` + imports ``pkg.sub.mod``. + (Contributed by Gregory P. Smith in :gh:`83065`.) + New modules =========== -* None yet. +math.integer +------------ + +This module provides access to the mathematical functions for integer +arguments (:pep:`791`). +(Contributed by Serhiy Storchaka in :gh:`81313`.) Improved modules ================ +argparse +-------- + +* The :class:`~argparse.BooleanOptionalAction` action supports now single-dash + long options and alternate prefix characters. + (Contributed by Serhiy Storchaka in :gh:`138525`.) + +* Changed the *suggest_on_error* parameter of :class:`argparse.ArgumentParser` to + default to ``True``. This enables suggestions for mistyped arguments by default. + (Contributed by Jakob Schluse in :gh:`140450`.) + +* Added backtick markup support in description and epilog text to highlight + inline code when color output is enabled. + (Contributed by Savannah Ostrowski in :gh:`142390`.) + + +array +----- + +* Support the :c:expr:`float complex` and :c:expr:`double complex` C types: + formatting characters ``'F'`` and ``'D'`` respectively. + (Contributed by Sergey B Kirpichev in :gh:`146151`.) + +* Support half-floats (16-bit IEEE 754 binary interchange format): formatting + character ``'e'``. + (Contributed by Sergey B Kirpichev in :gh:`146238`.) + + +ast +--- + +* Add *color* parameter to :func:`~ast.dump`. + If ``True``, the returned string is syntax highlighted using ANSI escape + sequences. + If ``False`` (the default), colored output is always disabled. + (Contributed by Stan Ulbrych in :gh:`148981`.) + +* The :ref:`command-line ` output is now syntax highlighted by default. + This can be :ref:`controlled using environment variables `. + (Contributed by Stan Ulbrych in :gh:`148981`.) + + +asyncio +------- + +* Added :meth:`TaskGroup.cancel ` to allow early + termination of a task group, for instance, when the goal of the tasks has + been achieved or their services are no longer needed. + Previously this would involve unintuitive boilerplate such as an extra task + raising a custom exception which is then suppressed as it exits the task group. + (Contributed by John Belmonte in :gh:`127214`.) + + +base64 +------ + +* Added the *pad* parameter in :func:`~base64.z85encode`. + (Contributed by Hauke Dämpfling in :gh:`143103`.) + +* Added the *padded* parameter in + :func:`~base64.b32encode`, :func:`~base64.b32decode`, + :func:`~base64.b32hexencode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b64encode`, :func:`~base64.b64decode`, + :func:`~base64.urlsafe_b64encode`, and :func:`~base64.urlsafe_b64decode`. + (Contributed by Serhiy Storchaka in :gh:`73613`.) + +* Added the *wrapcol* parameter in :func:`~base64.b16encode`, + :func:`~base64.b32encode`, :func:`~base64.b32hexencode`, + :func:`~base64.b64encode`, :func:`~base64.b85encode`, and + :func:`~base64.z85encode`. + (Contributed by Serhiy Storchaka in :gh:`143214` and :gh:`146431`.) + +* Added the *ignorechars* parameter in :func:`~base64.b16decode`, + :func:`~base64.b32decode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b64decode`, :func:`~base64.b85decode`, and + :func:`~base64.z85decode`. + (Contributed by Serhiy Storchaka in :gh:`144001` and :gh:`146431`.) + +* Added the *canonical* parameter in + :func:`~base64.b32decode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b64decode`, :func:`~base64.urlsafe_b64decode`, + :func:`~base64.a85decode`, :func:`~base64.b85decode`, and + :func:`~base64.z85decode`, + to reject encodings with non-zero padding bits or other non-canonical + forms. + (Contributed by Gregory P. Smith in :gh:`146311`.) + + +binascii +-------- + +* Added functions for Base32 encoding: + + - :func:`~binascii.b2a_base32` and :func:`~binascii.a2b_base32` + + (Contributed by James Seo in :gh:`146192`.) + +* Added functions for Ascii85, Base85, and Z85 encoding: + + - :func:`~binascii.b2a_ascii85` and :func:`~binascii.a2b_ascii85` + - :func:`~binascii.b2a_base85` and :func:`~binascii.a2b_base85` + + (Contributed by James Seo and Serhiy Storchaka in :gh:`101178`.) + +* Added the *padded* parameter in + :func:`~binascii.b2a_base32`, :func:`~binascii.a2b_base32`, + :func:`~binascii.b2a_base64`, and :func:`~binascii.a2b_base64`. + (Contributed by Serhiy Storchaka in :gh:`73613`.) + +* Added the *wrapcol* parameter in :func:`~binascii.b2a_base64`. + (Contributed by Serhiy Storchaka in :gh:`143214`.) + +* Added the *alphabet* parameter in :func:`~binascii.b2a_base64` and + :func:`~binascii.a2b_base64`. + (Contributed by Serhiy Storchaka in :gh:`145980`.) + +* Added the *ignorechars* parameter in :func:`~binascii.a2b_hex`, + :func:`~binascii.unhexlify`, and :func:`~binascii.a2b_base64`. + (Contributed by Serhiy Storchaka in :gh:`144001` and :gh:`146431`.) + +* Added the *canonical* parameter in :func:`~binascii.a2b_base64`, + to reject encodings with non-zero padding bits. + (Contributed by Gregory P. Smith in :gh:`146311`.) + + +calendar +-------- + +* Calendar pages generated by the :class:`calendar.HTMLCalendar` class now support + dark mode and have been migrated to the HTML5 standard for improved accessibility. + (Contributed by Jiahao Li and Hugo van Kemenade in :gh:`137634`.) + +* The :mod:`calendar`'s :ref:`command-line ` HTML output now + accepts the year-month option: ``python -m calendar -t html 2009 06``. + (Contributed by Pål Grønås Drange in :gh:`140212`.) + + +collections +----------- + +* Added :meth:`!collections.Counter.__xor__` and + :meth:`!collections.Counter.__ixor__` to compute the symmetric difference + between :class:`~collections.Counter` objects. + (Contributed by Raymond Hettinger in :gh:`138682`.) + + +concurrent.futures +------------------ + +* Improved error reporting when a child process in a + :class:`concurrent.futures.ProcessPoolExecutor` terminates abruptly. + The resulting traceback will now tell you the PID and exit code of the + terminated process. + (Contributed by Jonathan Berg in :gh:`139486`.) + + +contextlib +---------- + +* Added support for arbitrary descriptors :meth:`!__enter__`, + :meth:`!__exit__`, :meth:`!__aenter__`, and :meth:`!__aexit__` in + :class:`~contextlib.ExitStack` and :class:`contextlib.AsyncExitStack`, for + consistency with the :keyword:`with` and :keyword:`async with` statements. + (Contributed by Serhiy Storchaka in :gh:`144386`.) + +* :class:`~contextlib.ContextDecorator` and + :class:`~contextlib.AsyncContextDecorator` (and therefore + :func:`~contextlib.contextmanager` and :func:`~contextlib.asynccontextmanager` + used as decorators) now detect generator functions, coroutine functions, and + asynchronous generator functions and keep the context manager open across + iteration or await. Previously the context manager exited as soon as the + generator or coroutine object was created. + (Contributed by Alex Grönholm & Gregory P. Smith in :gh:`125862`.) + + +dataclasses +----------- + +* Annotations for generated ``__init__`` methods no longer include internal + type names. + + dbm --- @@ -290,11 +889,6 @@ dbm which allow to recover unused free space previously occupied by deleted entries. (Contributed by Andrea Oliveri in :gh:`134004`.) -* Add the ``'m'`` flag for :func:`dbm.gnu.open` which allows to disable - the use of :manpage:`mmap(2)`. - This may harm performance, but improve crash tolerance. - (Contributed by Serhiy Storchaka in :gh:`66234`.) - difflib ------- @@ -312,6 +906,37 @@ difflib (Contributed by Jiahao Li in :gh:`134580`.) +faulthandler +------------ + +* Added the *max_threads* parameter in :func:`faulthandler.enable`, + :func:`faulthandler.dump_traceback`, :func:`faulthandler.dump_traceback_later`, + and :func:`faulthandler.register`. + (Contributed by Eric Froemling in :gh:`149085`.) + + +email +----- + +* Email generators now raise an error when an :class:`.EmailMessage` cannot be + accurately flattened due to a non-ASCII email address (mailbox) in an address + header. Options for supporting Email Address Internationalization (EAI) are + discussed in :attr:`.EmailPolicy.utf8`. + (Contributed by R David Murray and Mike Edmunds in :gh:`122540`.) + + +functools +--------- + +* :func:`~functools.singledispatchmethod` now supports non-:term:`descriptor` + callables. + (Contributed by Serhiy Storchaka in :gh:`140873`.) + +* :func:`~functools.singledispatchmethod` now dispatches on the second argument + if it wraps a regular method and is called as a class attribute. + (Contributed by Bartosz Sławecki in :gh:`143535`.) + + hashlib ------- @@ -333,21 +958,53 @@ http.client (Contributed by Alexander Enrique Urieles Nieto in :gh:`131724`.) -http.cookies ------------- +http.server +----------- -* Allow '``"``' double quotes in cookie values. - (Contributed by Nick Burns and Senthil Kumaran in :gh:`92936`.) +* The logging of :mod:`~http.server.BaseHTTPRequestHandler`, + as used by the :ref:`command-line interface `, + is colored by default. + This can be controlled with :ref:`environment variables + `. + (Contributed by Hugo van Kemenade in :gh:`146292`.) + +* Added :attr:`~http.server.SimpleHTTPRequestHandler.default_content_type` + and the :option:`--content-type ` command-line + option to allow customizing the default ``Content-Type`` header + for files with unknown extensions. + (Contributed by John Comeau and Hugo van Kemenade in :gh:`113471`.) + + +inspect +------- + +* Add parameters *inherit_class_doc* and *fallback_to_class_doc* + for :func:`~inspect.getdoc`. + (Contributed by Serhiy Storchaka in :gh:`132686`.) + +json +---- + +* Add the *array_hook* parameter to :func:`~json.load` and + :func:`~json.loads` functions: + allow a callback for JSON literal array types to customize Python lists in + the resulting decoded object. Passing combined :class:`frozendict` to + *object_pairs_hook* param and :class:`tuple` to ``array_hook`` will yield a + deeply nested immutable Python structure representing the JSON data. + (Contributed by Joao S. O. Bueno in :gh:`146440`.) locale ------ * :func:`~locale.setlocale` now supports language codes with ``@``-modifiers. - ``@``-modifier are no longer silently removed in :func:`~locale.getlocale`, + ``@``-modifiers are no longer silently removed in :func:`~locale.getlocale`, but included in the language code. (Contributed by Serhiy Storchaka in :gh:`137729`.) +* Undeprecate the :func:`locale.getdefaultlocale` function. + (Contributed by Victor Stinner in :gh:`130796`.) + math ---- @@ -359,6 +1016,20 @@ math (Contributed by Bénédikt Tran in :gh:`135853`.) +mimetypes +--------- + +* Add more MIME types. + (Contributed by Benedikt Johannes, Charlie Lin, Foolbar, Gil Forcada and + John Franey + in :gh:`144217`, :gh:`145720`, :gh:`140937`, :gh:`139959`, :gh:`145698`, + :gh:`145718` and :gh:`144213`.) +* Rename ``application/x-texinfo`` to ``application/texinfo``. + (Contributed by Charlie Lin in :gh:`140165`.) +* Changed the MIME type for ``.ai`` files to ``application/pdf``. + (Contributed by Stan Ulbrych in :gh:`141239`.) + + mmap ---- @@ -367,6 +1038,19 @@ mmap not be duplicated. (Contributed by Serhiy Storchaka in :gh:`78502`.) +* Added the :meth:`mmap.mmap.set_name` method + to annotate an anonymous memory mapping + if Linux kernel supports :manpage:`PR_SET_VMA_ANON_NAME ` (Linux 5.17 or newer). + (Contributed by Donghee Na in :gh:`142419`.) + + +os +-- + +* Add :func:`os.statx` on Linux kernel versions 4.11 and later with + glibc versions 2.28 and later. + (Contributed by Jeffrey Bosboom and Victor Stinner in :gh:`83714`.) + os.path ------- @@ -381,6 +1065,57 @@ os.path (Contributed by Petr Viktorin for :cve:`2025-4517`.) +pdb +--- + +* Use the new interactive shell as the default input shell for :mod:`pdb`. + (Contributed by Tian Gao in :gh:`145379`.) + + +pickle +------ + +* Add support for pickling private methods and nested classes. + (Contributed by Zackery Spytz and Serhiy Storchaka in :gh:`77188`.) + + +pickletools +----------- + +* The output of the :mod:`pickletools` command-line interface is colored by + default. This can be controlled with + :ref:`environment variables `. + (Contributed by Hugo van Kemenade in :gh:`149026`.) + + +pprint +------ + +* Add an *expand* keyword argument for :func:`pprint.pprint`, + :func:`pprint.pformat`, :func:`pprint.pp`. If true, the output will be + formatted similar to pretty-printed :func:`json.dumps` when + *indent* is supplied. + (Contributed by Stefan Todoran, Semyon Moroz and Hugo van Kemenade in + :gh:`112632`.) + +* Add t-string support to :mod:`pprint`. + (Contributed by Loïc Simon and Hugo van Kemenade in :gh:`134551`.) + + +re +-- + +* :func:`re.prefixmatch` and a corresponding :meth:`re.Pattern.prefixmatch` + have been added as alternate, more explicit names for the existing + and now :term:`soft deprecated` + :func:`re.match` and :meth:`re.Pattern.match` APIs. These are intended + to be used to alleviate confusion around what *match* means by following the + Zen of Python's *"Explicit is better than implicit"* mantra. Most other + language regular expression libraries use an API named *match* to mean what + Python has always called *search*. + (Contributed by Gregory P. Smith in :gh:`86519`.) + + resource -------- @@ -396,6 +1131,16 @@ shelve * Added new :meth:`!reorganize` method to :mod:`shelve` used to recover unused free space previously occupied by deleted entries. (Contributed by Andrea Oliveri in :gh:`134004`.) +* Add support for custom serialization and deserialization functions + in the :mod:`shelve` module. + (Contributed by Furkan Onder in :gh:`99631`.) + + +socket +------ + +* Add constants for the ISO-TP CAN protocol. + (Contributed by Patrick Menschel and Stefan Tatschner in :gh:`86819`.) sqlite3 @@ -403,39 +1148,42 @@ sqlite3 * The :ref:`command-line interface ` has several new features: - * SQL keyword completion on . - (Contributed by Long Tan in :gh:`133393`.) + * SQL keyword completion on . + (Contributed by Long Tan in :gh:`133393`.) - * Prompts, error messages, and help text are now colored. - This is enabled by default, see :ref:`using-on-controlling-color` for - details. - (Contributed by Stan Ulbrych and Łukasz Langa in :gh:`133461`.) + * Prompts, error messages, and help text are now colored. + This is enabled by default, see :ref:`using-on-controlling-color` for + details. + (Contributed by Stan Ulbrych and Łukasz Langa in :gh:`133461`.) + + * Table, index, trigger, view, column, function, and schema completion on . + (Contributed by Long Tan in :gh:`136101`.) ssl --- * Indicate through :data:`ssl.HAS_PSK_TLS13` whether the :mod:`ssl` module - supports "External PSKs" in TLSv1.3, as described in RFC 9258. + supports "External PSKs" in TLSv1.3, as described in :rfc:`9258`. (Contributed by Will Childs-Klein in :gh:`133624`.) * Added new methods for managing groups used for SSL key agreement - * :meth:`ssl.SSLContext.set_groups` sets the groups allowed for doing - key agreement, extending the previous - :meth:`ssl.SSLContext.set_ecdh_curve` method. - This new API provides the ability to list multiple groups and - supports fixed-field and post-quantum groups in addition to ECDH - curves. This method can also be used to control what key shares - are sent in the TLS handshake. - * :meth:`ssl.SSLSocket.group` returns the group selected for doing key - agreement on the current connection after the TLS handshake completes. - This call requires OpenSSL 3.2 or later. - * :meth:`ssl.SSLContext.get_groups` returns a list of all available key - agreement groups compatible with the minimum and maximum TLS versions - currently set in the context. This call requires OpenSSL 3.5 or later. + * :meth:`ssl.SSLContext.set_groups` sets the groups allowed for doing + key agreement, extending the previous + :meth:`ssl.SSLContext.set_ecdh_curve` method. + This new API provides the ability to list multiple groups and + supports fixed-field and post-quantum groups in addition to ECDH + curves. This method can also be used to control what key shares + are sent in the TLS handshake. + * :meth:`ssl.SSLSocket.group` returns the group selected for doing key + agreement on the current connection after the TLS handshake completes. + This call requires OpenSSL 3.2 or later. + * :meth:`ssl.SSLContext.get_groups` returns a list of all available key + agreement groups compatible with the minimum and maximum TLS versions + currently set in the context. This call requires OpenSSL 3.5 or later. - (Contributed by Ron Frederick in :gh:`136306`.) + (Contributed by Ron Frederick in :gh:`136306`.) * Added a new method :meth:`ssl.SSLContext.set_ciphersuites` for setting TLS 1.3 ciphers. For TLS 1.2 or earlier, :meth:`ssl.SSLContext.set_ciphers` should @@ -459,7 +1207,30 @@ ssl selected for the server to complete the TLS handshake on the current connection. This call requires OpenSSL 3.5 or later. - (Contributed by Ron Frederick in :gh:`138252`.) + (Contributed by Ron Frederick in :gh:`138252`.) + + +subprocess +---------- + +* :meth:`subprocess.Popen.wait`: when ``timeout`` is not ``None`` and the + platform supports it, an efficient event-driven mechanism is used to wait for + process termination: + + - Linux >= 5.3 uses :func:`os.pidfd_open` + :func:`select.poll`. + - macOS and other BSD variants use :func:`select.kqueue` + ``KQ_FILTER_PROC`` + ``KQ_NOTE_EXIT``. + - Windows keeps using ``WaitForSingleObject`` (unchanged). + + If none of these mechanisms are available, the function falls back to the + traditional busy loop (non-blocking call and short sleeps). + (Contributed by Giampaolo Rodola in :gh:`83069`.) + + +symtable +-------- + +* Add :meth:`symtable.Function.get_cells` and :meth:`symtable.Symbol.is_cell` methods. + (Contributed by Yashp002 in :gh:`143504`.) sys @@ -469,6 +1240,19 @@ sys (Contributed by Klaus Zimmermann in :gh:`137476`.) +sys.monitoring +-------------- + +* The :ref:`other events ` + (:monitoring-event:`PY_THROW`, :monitoring-event:`PY_UNWIND`, + :monitoring-event:`RAISE`, :monitoring-event:`EXCEPTION_HANDLED`, and + :monitoring-event:`RERAISE`) can now be turned on and disabled on a per code + object basis. Returning :data:`~sys.monitoring.DISABLE` from a callback for + one of these events disables the event for the entire code object (for the + current tool), rather than raising :exc:`ValueError` as in prior versions. + (Contributed by Gabriele N. Tornetta in :gh:`146182`.) + + tarfile ------- @@ -495,15 +1279,195 @@ tarfile (Contributed by Christoph Walcher in :gh:`57911`.) -types +threading +--------- + +* Added :class:`~threading.serialize_iterator`, + :func:`~threading.synchronized_iterator`, + and :func:`~threading.concurrent_tee` to support concurrent access to + generators and iterators. + (Contributed by Raymond Hettinger in :gh:`124397`.) + + +timeit ------ +* The output of the :mod:`timeit` command-line interface is colored by default. + This can be controlled with + :ref:`environment variables `. + (Contributed by Hugo van Kemenade in :gh:`146609`.) +* The command-line interface now colorizes error tracebacks + by default. This can be controlled with + :ref:`environment variables `. + (Contributed by Yi Hong in :gh:`139374`.) + +* Make the target time of :meth:`timeit.Timer.autorange` configurable + and add ``--target-time`` option to the command-line interface. + (Contributed by Alessandro Cucci and Miikka Koskinen in :gh:`80642`.) + + +tkinter +------- + +* The :meth:`!tkinter.Text.search` method now supports two additional + arguments: *nolinestop* which allows the search to + continue across line boundaries; + and *strictlimits* which restricts the search to within the specified range. + (Contributed by Rihaan Meher in :gh:`130848`.) + +* A new method :meth:`!tkinter.Text.search_all` has been introduced. + This method allows for searching for all matches of a pattern + using Tcl's ``-all`` and ``-overlap`` options. + (Contributed by Rihaan Meher in :gh:`130848`.) + +* Added new methods :meth:`!pack_content`, :meth:`!place_content` and + :meth:`!grid_content` which use Tk commands with new names (introduced + in Tk 8.6) instead of :meth:`!*_slaves` methods which use Tk commands + with outdated names. + (Contributed by Serhiy Storchaka in :gh:`143754`.) + +* Added :class:`!Event` attributes :attr:`!user_data` for Tk virtual events + and :attr:`!detail` for ``Enter``, ``Leave``, ``FocusIn``, ``FocusOut``, + and ``ConfigureRequest`` events. + (Contributed by Matthias Kievernagel and Serhiy Storchaka in :gh:`47655`.) + + +tokenize +-------- + +* The output of the :mod:`tokenize` :ref:`command-line interface + ` is colored by default. This can be controlled with + :ref:`environment variables `. + (Contributed by Hugo van Kemenade in :gh:`148991`.) + + +.. _whatsnew315-tomllib-1-1-0: + +tomllib +------- + +* The :mod:`tomllib` module now supports TOML 1.1.0. + This is a backwards compatible update, meaning that all valid TOML 1.0.0 + documents are parsed the same way. + + The changes, according to the `official TOML changelog`_, are: + + - Allow newlines and trailing commas in inline tables. + + Previously an inline table had to be on a single line and couldn't end + with a trailing comma. This is now relaxed so that the following is valid: + + .. code-block:: toml + + tbl = { + key = "a string", + moar-tbl = { + key = 1, + }, + } + + - Add ``\xHH`` notation to basic strings for codepoints under 255, + and the ``\e`` escape for the escape character: + + .. code-block:: toml + + null = "null byte: \x00; letter a: \x61" + csi = "\e[" + + - Seconds in datetime and time values are now optional. + The following are now valid: + + .. code-block:: toml + + dt = 2010-02-03 14:15 + t = 14:15 + + (Contributed by Taneli Hukkinen in :gh:`142956`.) + +.. _official TOML changelog: https://github.com/toml-lang/toml/blob/main/CHANGELOG.md + + +types +----- + * Expose the write-through :func:`locals` proxy type as :data:`types.FrameLocalsProxyType`. This represents the type of the :attr:`frame.f_locals` attribute, as described in :pep:`667`. +typing +------ + +.. _whatsnew315-typeform: + +* :pep:`747`: Add :data:`~typing.TypeForm`, a new special form for annotating + values that are themselves type expressions. + ``TypeForm[T]`` means "a type form object describing ``T`` (or a type + assignable to ``T``)". At runtime, ``TypeForm(x)`` simply returns ``x``, + which allows explicit annotation of type-form values without changing + behavior. + + This helps libraries that accept user-provided type expressions + (for example ``int``, ``str | None``, :class:`~typing.TypedDict` + classes, or ``list[int]``) expose precise signatures: + + .. code-block:: python + + from typing import Any, TypeForm + + def cast[T](typ: TypeForm[T], value: Any) -> T: ... + + (Contributed by Jelle Zijlstra in :gh:`145033`.) + +* Code like ``class ExtraTypeVars(P1[S], Protocol[T, T2]): ...`` now raises + a :exc:`TypeError`, because ``S`` is not listed in ``Protocol`` parameters. + (Contributed by Nikita Sobolev in :gh:`137191`.) + +* Code like ``class B2(A[T2], Protocol[T1, T2]): ...`` now correctly handles + type parameters order: it is ``(T1, T2)``, not ``(T2, T1)`` + as it was incorrectly inferred in runtime before. + (Contributed by Nikita Sobolev in :gh:`137191`.) + +* :pep:`800`: Add :deco:`typing.disjoint_base`, a new decorator marking a class + as a disjoint base. This is an advanced feature primarily intended to allow + type checkers to faithfully reflect the runtime semantics of types defined + as builtins or in compiled extensions. If a class ``C`` is a disjoint base, then + child classes of that class cannot inherit from other disjoint bases that are + not parent or child classes of ``C``. (Contributed by Jelle Zijlstra in :gh:`148639`.) + +* :class:`~typing.TypeVarTuple` now accepts ``bound``, ``covariant``, + ``contravariant``, and ``infer_variance`` keyword arguments, matching the + interface of :class:`~typing.TypeVar` and :class:`~typing.ParamSpec`. + ``bound`` semantics remain undefined in the specification. + + +unicodedata +----------- + +* The Unicode database has been updated to Unicode 17.0.0. + +* Add :func:`unicodedata.isxidstart` and :func:`unicodedata.isxidcontinue` + functions to check whether a character can start or continue a + `Unicode Standard Annex #31 `_ identifier. + (Contributed by Stan Ulbrych in :gh:`129117`.) + +* Add the :func:`~unicodedata.iter_graphemes` + function to iterate over grapheme clusters according to rules defined in + `Unicode Standard Annex #29, "Unicode Text Segmentation" + `_. + Add :func:`~unicodedata.grapheme_cluster_break`, + :func:`~unicodedata.indic_conjunct_break` and + :func:`~unicodedata.extended_pictographic` functions to get the properties + of the character which are related to the above algorithm. + (Contributed by Serhiy Storchaka and Guillaume Sanchez in :gh:`74902`.) + +* Add :func:`~unicodedata.block` function to return the `Unicode block + `_ + assigned to a character. + (Contributed by Stan Ulbrych in :gh:`66802`.) + + unittest -------- @@ -512,6 +1476,78 @@ unittest (Contributed by Garry Cairns in :gh:`134567`.) +urllib.parse +------------ + +* Add the *missing_as_none* parameter to :func:`~urllib.parse.urlsplit`, + :func:`~urllib.parse.urlparse` and :func:`~urllib.parse.urldefrag` functions. + Add the *keep_empty* parameter to :func:`~urllib.parse.urlunsplit` and + :func:`~urllib.parse.urlunparse` functions. + This allows to distinguish between empty and not defined URI components + and preserve empty components. + (Contributed by Serhiy Storchaka in :gh:`67041`.) + + +venv +---- + +* On POSIX platforms, platlib directories will be created if needed when + creating virtual environments, instead of using ``lib64 -> lib`` symlink. + This means purelib and platlib of virtual environments no longer share the + same ``lib`` directory on platforms where :data:`sys.platlibdir` is not + equal to ``lib``. + (Contributed by Rui Xi in :gh:`133951`.) + + +warnings +-------- + +* Improve filtering by module in :func:`warnings.warn_explicit` if no *module* + argument is passed. + It now tests the module regular expression in the warnings filter not only + against the filename with ``.py`` stripped, but also against module names + constructed starting from different parent directories of the filename + (with ``/__init__.py``, ``.py`` and, on Windows, ``.pyw`` stripped). + (Contributed by Serhiy Storchaka in :gh:`135801`.) + + +wave +---- + +* Added support for IEEE floating-point WAVE audio + (``WAVE_FORMAT_IEEE_FLOAT``) in :mod:`wave`. + +* Added :meth:`wave.Wave_read.getformat`, :meth:`wave.Wave_write.getformat`, + and :meth:`wave.Wave_write.setformat` for explicit frame format handling. + +* :meth:`wave.Wave_write.setparams` accepts both 7-item tuples including + ``format`` and 6-item tuples for backwards compatibility (defaulting to + ``WAVE_FORMAT_PCM``). + +* ``WAVE_FORMAT_IEEE_FLOAT`` output now includes a ``fact`` chunk, + as required for non-PCM WAVE formats. + +(Contributed by Lionel Koenig and Michiel W. Beijen in :gh:`60729`.) + + +xml.parsers.expat +----------------- + +* Add :meth:`~xml.parsers.expat.xmlparser.SetAllocTrackerActivationThreshold` + and :meth:`~xml.parsers.expat.xmlparser.SetAllocTrackerMaximumAmplification` + to :ref:`xmlparser ` objects to tune protections against + disproportional amounts of dynamic memory usage from within an Expat parser. + (Contributed by Bénédikt Tran in :gh:`90949`.) + +* Add :meth:`~xml.parsers.expat.xmlparser.SetBillionLaughsAttackProtectionActivationThreshold` + and :meth:`~xml.parsers.expat.xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification` + to :ref:`xmlparser ` objects to tune protections against + `billion laughs`_ attacks. + (Contributed by Bénédikt Tran in :gh:`90949`.) + + .. _billion laughs: https://en.wikipedia.org/wiki/Billion_laughs_attack + + zlib ---- @@ -527,36 +1563,152 @@ zlib Optimizations ============= -module_name ------------ - -* TODO +* ``mimalloc`` is now used as the default allocator for + raw memory allocations such as via :c:func:`PyMem_RawMalloc` + for better performance on :term:`free-threaded builds `. + (Contributed by Kumar Aditya in :gh:`144914`.) +base64 & binascii +----------------- -Deprecated -========== +* CPython's underlying base64 implementation now encodes 2x faster and decodes 3x + faster thanks to simple CPU pipelining optimizations. + (Contributed by Gregory P. Smith and Serhiy Storchaka in :gh:`143262`.) -hashlib -------- +* Implementation for Ascii85, Base85, and Z85 encoding has been rewritten in C. + Encoding and decoding is now two orders of magnitude faster and consumes + two orders of magnitude less memory. + (Contributed by James Seo and Serhiy Storchaka in :gh:`101178`.) -* In hash function constructors such as :func:`~hashlib.new` or the - direct hash-named constructors such as :func:`~hashlib.md5` and - :func:`~hashlib.sha256`, their optional initial data parameter could - also be passed a keyword argument named ``data=`` or ``string=`` in - various :mod:`hashlib` implementations. - - Support for the ``string`` keyword argument name is now deprecated and - is slated for removal in Python 3.19. Prefer passing the initial data as - a positional argument for maximum backwards compatibility. - - (Contributed by Bénédikt Tran in :gh:`134978`.) +* Implementation for Base32 has been rewritten in C. + Encoding and decoding is now two orders of magnitude faster. + (Contributed by James Seo in :gh:`146192`.) -.. Add deprecations above alphabetically, not here at the end. +csv +--- + +* :meth:`csv.Sniffer.sniff` delimiter detection is now up to 1.6x faster. + (Contributed by Maurycy Pawłowski-Wieroński in :gh:`137628`.) + + +.. _whatsnew315-jit: + +Upgraded JIT compiler +--------------------- + +Results from the `pyperformance `__ +benchmark suite report +`8-9% `__ +geometric mean performance improvement for the JIT over the standard CPython +interpreter built with all optimizations enabled on x86-64 Linux. On AArch64 +macOS, the JIT has a +`12-13% `__ +speedup over the :ref:`tail calling interpreter ` +with all optimizations enabled. The speedups for JIT +builds versus no JIT builds range from roughly 15% slowdown to over +100% speedup (ignoring the ``unpack_sequence`` microbenchmark) on +x86-64 Linux and AArch64 macOS systems. + +.. attention:: + These results are not yet final. + +The major upgrades to the JIT are: + +* LLVM 21 build-time dependency +* New tracing frontend +* Basic register allocation in the JIT +* More JIT optimizations +* Better machine code generation + +.. rubric:: LLVM 21 build-time dependency + +The JIT compiler now uses LLVM 21 for build-time stencil generation. As +always, LLVM is only needed when building CPython with the JIT enabled; +end users running Python do not need LLVM installed. Instructions for +installing LLVM can be found in the `JIT compiler documentation +`__ +for all supported platforms. +(Contributed by Savannah Ostrowski in :gh:`140973`.) + +.. rubric:: A new tracing frontend + +The JIT compiler now supports significantly more bytecode operations and +control flow than in Python 3.14, enabling speedups on a wider variety of +code. For example, simple Python object creation is now understood by the +3.15 JIT compiler. Overloaded operations and generators are also partially +supported. This was made possible by an overhauled JIT tracing frontend +that records actual execution paths through code, rather than estimating +them as the previous implementation did. +(Contributed by Ken Jin in :gh:`139109`. Support for Windows added by +Mark Shannon in :gh:`141703`.) + +.. rubric:: Basic register allocation in the JIT + +A basic form of register allocation has been added to the JIT compiler's +optimizer. This allows the JIT compiler to avoid certain stack operations +altogether and instead operate on registers. This allows the JIT to produce +more efficient traces by avoiding reads and writes to memory. +(Contributed by Mark Shannon in :gh:`135379`.) + +.. rubric:: More JIT optimizations + +More `constant-propagation `__ +is now performed. This means when the JIT compiler detects that certain user +code results in constants, the code can be simplified by the JIT. +(Contributed by Ken Jin and Savannah Ostrowski in :gh:`132732`.) + +:term:`Reference count`\ s are avoided whenever it is safe to do so. This generally +reduces the cost of most operations in Python. +(Contributed by Ken Jin, Donghee Na, Zheao Li, Hai Zhu, Savannah Ostrowski, +Reiden Ong, Noam Cohen, Tomas Roun, PuQing, Cajetan Rodrigues, and Sacul in :gh:`134584`.) + +By tracking unique references to objects, the JIT optimizer can now eliminate +reference count updates and perform inplace operations on ints and floats. +(Contributed by Reiden Ong, and Pieter Eendebak in :gh:`143414` and :gh:`146306`.) + +The JIT optimizer now supports significantly more operations than in 3.14. +(Contributed by Kumar Aditya, Ken Jin, Jiahao Li, and Sacul in :gh:`131798`.) + +.. rubric:: Better machine code generation + +The JIT compiler's machine code generator now produces better machine code +for x86-64 and AArch64 macOS and Linux targets. In general, users should +experience lower memory usage for generated machine code and more efficient +machine code versus 3.14. +(Contributed by Brandt Bucher in :gh:`136528` and :gh:`135905`. +Implementation for AArch64 contributed by Mark Shannon in :gh:`139855`. +Additional optimizations for AArch64 contributed by Mark Shannon and +Diego Russo in :gh:`140683` and :gh:`142305`.) + +.. rubric:: Maintainability + +The JIT optimizer's operations have been simplified. +This was made possible by a refactoring of JIT data structures. +(Contributed by Zhongtian Zheng in :gh:`148211` and Hai Zhu in :gh:`143421`.) + Removed -======= +======== + +ast +--- + +* The constructors of :ref:`AST nodes ` now raise a :exc:`TypeError` + when a required argument is omitted or when a keyword argument that does not + map to a field on the AST node is passed. These cases had previously raised a + :exc:`DeprecationWarning` since Python 3.13. + (Contributed by Brian Schubert and Jelle Zijlstra in :gh:`137600` and :gh:`105858`.) + + +collections.abc +--------------- + +* :class:`collections.abc.ByteString` has been removed from + ``collections.abc.__all__``. :class:`!collections.abc.ByteString` has been + deprecated since Python 3.12, and is scheduled for removal in Python 3.17. + ctypes ------ @@ -566,6 +1718,15 @@ ctypes (Contributed by Bénédikt Tran in :gh:`133866`.) +datetime +-------- + +* :meth:`~datetime.datetime.strptime` now raises :exc:`ValueError` when the + format string contains ``%d`` (day of month) without a year directive. + This has been deprecated since Python 3.13. + (Contributed by Stan Ulbrych and Gregory P. Smith in :gh:`70647`.) + + glob ---- @@ -584,6 +1745,14 @@ http.server (Contributed by Bénédikt Tran in :gh:`133810`.) +importlib.resources +------------------- + +* Removed deprecated ``package`` parameter + from :func:`importlib.resources.files` function. + (Contributed by Semyon Moroz in :gh:`138044`.) + + pathlib ------- @@ -623,9 +1792,21 @@ threading (Contributed by Bénédikt Tran in :gh:`134087`.) +types +----- + +* Removed deprecated in :pep:`626` since Python 3.12 + :attr:`!codeobject.co_lnotab` from :class:`types.CodeType`. + (Contributed by Nikita Sobolev in :gh:`134690`.) + + typing ------ +* :class:`typing.ByteString` has been removed from ``typing.__all__``. + :class:`!typing.ByteString` has been deprecated since Python 3.9, and is + scheduled for removal in Python 3.17. + * The undocumented keyword argument syntax for creating :class:`~typing.NamedTuple` classes (for example, ``Point = NamedTuple("Point", x=int, y=int)``) is no longer supported. @@ -633,19 +1814,13 @@ typing (Contributed by Bénédikt Tran in :gh:`133817`.) * Using ``TD = TypedDict("TD")`` or ``TD = TypedDict("TD", None)`` to - construct a :class:`~typing.TypedDict` type with zero field is no + construct a :class:`~typing.TypedDict` type with zero fields is no longer supported. Use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})`` instead. (Contributed by Bénédikt Tran in :gh:`133823`.) -* Code like ``class ExtraTypeVars(P1[S], Protocol[T, T2]): ...`` now raises - a :exc:`TypeError`, because ``S`` is not listed in ``Protocol`` parameters. - (Contributed by Nikita Sobolev in :gh:`137191`.) - -* Code like ``class B2(A[T2], Protocol[T1, T2]): ...`` now correctly handles - type parameters order: it is ``(T1, T2)``, not ``(T2, T1)`` - as it was incorrectly infered in runtime before. - (Contributed by Nikita Sobolev in :gh:`137191`.) +* Deprecated :func:`!typing.no_type_check_decorator` has been removed. + (Contributed by Nikita Sobolev in :gh:`133601`.) wave @@ -665,20 +1840,159 @@ zipimport (Contributed by Jiahao Li in :gh:`133656`.) -Porting to Python 3.15 -====================== +Deprecated +========== -This section lists previously described changes and other bugfixes -that may require changes to your code. +New deprecations +---------------- + +* :mod:`ast` + + * Creating instances of abstract AST nodes (such as :class:`ast.AST` + or :class:`!ast.expr`) is deprecated and will raise an error in Python 3.20. + + (Contributed by Brian Schubert in :gh:`116021`.) + +* :mod:`base64`: + + * Accepting the ``+`` and ``/`` characters with an alternative alphabet in + :func:`~base64.b64decode` and :func:`~base64.urlsafe_b64decode` is now + deprecated. + In future Python versions they will be errors in the strict mode and + discarded in the non-strict mode. + (Contributed by Serhiy Storchaka in :gh:`125346`.) + +* CLI: + + * Deprecate :option:`-b` and :option:`!-bb` command-line options + and schedule them to become no-ops in Python 3.17. + These were primarily helpers for the Python 2 -> 3 transition. + Starting with Python 3.17, no :exc:`BytesWarning` will be raised + for these cases; use a type checker instead. + + (Contributed by Nikita Sobolev in :gh:`136355`.) + +* :mod:`collections.abc` + + * The following statements now cause ``DeprecationWarning``\ s to be emitted + at runtime: + + * ``from collections.abc import ByteString`` + * ``import collections.abc; collections.abc.ByteString``. + + ``DeprecationWarning``\ s were already emitted if + :class:`collections.abc.ByteString` was subclassed or used as the second + argument to :func:`isinstance` or :func:`issubclass`, but warnings were not + previously emitted if it was merely imported or accessed from the + :mod:`!collections.abc` module. -Build changes -============= +* :mod:`hashlib`: -* Removed implicit fallback to the bundled copy of the ``libmpdec`` library. - Now this should be explicitly enabled with :option:`--with-system-libmpdec` - set to ``no`` or with :option:`!--without-system-libmpdec`. - (Contributed by Sergey B Kirpichev in :gh:`115119`.) + * In hash function constructors such as :func:`~hashlib.new` or the + direct hash-named constructors such as :func:`~hashlib.md5` and + :func:`~hashlib.sha256`, the optional initial data parameter could + also be passed as a keyword argument named ``data=`` or ``string=`` in + various :mod:`hashlib` implementations. + + Support for the ``string`` keyword argument name is now deprecated and + is slated for removal in Python 3.19. Prefer passing the initial data as + a positional argument for maximum backwards compatibility. + + (Contributed by Bénédikt Tran in :gh:`134978`.) + + +* :mod:`re`: + + * :func:`re.match` and :meth:`re.Pattern.match` are now + :term:`soft deprecated` in favor of the new :func:`re.prefixmatch` and + :meth:`re.Pattern.prefixmatch` APIs, which have been added as alternate, + more explicit names. These are intended to be used to alleviate confusion + around what *match* means by following the Zen of Python's *"Explicit is + better than implicit"* mantra. Most other language regular expression + libraries use an API named *match* to mean what Python has always called + *search*. + + We **do not** plan to remove the older :func:`!match` name, as it has been + used in code for over 30 years. Code supporting older versions of Python + should continue to use :func:`!match`, while new code should prefer + :func:`!prefixmatch`. See :ref:`prefixmatch-vs-match`. + + (Contributed by Gregory P. Smith in :gh:`86519` and + Hugo van Kemenade in :gh:`148100`.) + + +* :mod:`struct`: + + * Calling the ``Struct.__new__()`` without required argument now is + deprecated and will be removed in Python 3.20. Calling + :meth:`~object.__init__` method on initialized :class:`~struct.Struct` + objects is deprecated and will be removed in Python 3.20. + + (Contributed by Sergey B Kirpichev and Serhiy Storchaka in :gh:`143715`.) + +* :mod:`typing`: + + * The following statements now cause ``DeprecationWarning``\ s to be emitted + at runtime: + + * ``from typing import ByteString`` + * ``import typing; typing.ByteString``. + + ``DeprecationWarning``\ s were already emitted if :class:`typing.ByteString` + was subclassed or used as the second argument to :func:`isinstance` or + :func:`issubclass`, but warnings were not previously emitted if it was + merely imported or accessed from the :mod:`!typing` module. + + +* ``__version__`` + + * The ``__version__``, ``version`` and ``VERSION`` attributes have been + deprecated in these standard library modules and will be removed in + Python 3.20. Use :py:data:`sys.version_info` instead. + + - :mod:`argparse` + - :mod:`csv` + - :mod:`ctypes` + - :mod:`!ctypes.macholib` + - :mod:`decimal` (use :data:`decimal.SPEC_VERSION` instead) + - :mod:`http.server` + - :mod:`imaplib` + - :mod:`ipaddress` + - :mod:`json` + - :mod:`logging` (``__date__`` also deprecated) + - :mod:`optparse` + - :mod:`pickle` + - :mod:`platform` + - :mod:`re` + - :mod:`socketserver` + - :mod:`tabnanny` + - :mod:`tarfile` + - :mod:`tkinter.font` + - :mod:`tkinter.ttk` + - :mod:`wsgiref.simple_server` + - :mod:`xml.etree.ElementTree` + - :mod:`!xml.sax.expatreader` + - :mod:`xml.sax.handler` + - :mod:`zlib` + + (Contributed by Hugo van Kemenade and Stan Ulbrych in :gh:`76007`.) + +.. Add deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/pending-removal-in-3.16.rst + +.. include:: ../deprecations/pending-removal-in-3.17.rst + +.. include:: ../deprecations/pending-removal-in-3.18.rst + +.. include:: ../deprecations/pending-removal-in-3.19.rst + +.. include:: ../deprecations/pending-removal-in-3.20.rst + +.. include:: ../deprecations/pending-removal-in-future.rst + +.. include:: ../deprecations/soft-deprecations.rst C API changes @@ -687,6 +2001,21 @@ C API changes New features ------------ +* Add :c:func:`PyArg_ParseArray` and :c:func:`PyArg_ParseArrayAndKeywords` + functions to parse arguments of functions using the :c:macro:`METH_FASTCALL` + calling convention. + (Contributed by Victor Stinner in :gh:`144175`.) + +* Add the following functions for the new :class:`frozendict` type: + + * :c:func:`PyAnyDict_Check` + * :c:func:`PyAnyDict_CheckExact` + * :c:func:`PyFrozenDict_Check` + * :c:func:`PyFrozenDict_CheckExact` + * :c:func:`PyFrozenDict_New` + + (Contributed by Victor Stinner in :gh:`141510`.) + * Add :c:func:`PySys_GetAttr`, :c:func:`PySys_GetAttrString`, :c:func:`PySys_GetOptionalAttr`, and :c:func:`PySys_GetOptionalAttrString` functions as replacements for :c:func:`PySys_GetObject`. @@ -701,59 +2030,80 @@ New features and :c:data:`Py_mod_abi`. (Contributed by Petr Viktorin in :gh:`137210`.) +.. _whatsnew315-pybyteswriter: + +* Implement :pep:`782`, the :ref:`PyBytesWriter API `. + Add functions: + + * :c:func:`PyBytesWriter_Create` + * :c:func:`PyBytesWriter_Discard` + * :c:func:`PyBytesWriter_FinishWithPointer` + * :c:func:`PyBytesWriter_FinishWithSize` + * :c:func:`PyBytesWriter_Finish` + * :c:func:`PyBytesWriter_Format` + * :c:func:`PyBytesWriter_GetData` + * :c:func:`PyBytesWriter_GetSize` + * :c:func:`PyBytesWriter_GrowAndUpdatePointer` + * :c:func:`PyBytesWriter_Grow` + * :c:func:`PyBytesWriter_Resize` + * :c:func:`PyBytesWriter_WriteBytes` + + (Contributed by Victor Stinner in :gh:`129813`.) + +* Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating + a module from a *spec* and *initfunc*. + (Contributed by Itamar Oren in :gh:`116146`.) + +* Add :c:func:`PyTuple_FromArray` to create a :class:`tuple` from an array. + (Contributed by Victor Stinner in :gh:`111489`.) + +* Add functions that are guaranteed to be safe for use in + :c:member:`~PyTypeObject.tp_traverse` handlers: + :c:func:`PyObject_GetTypeData_DuringGC`, + :c:func:`PyObject_GetItemData_DuringGC`, + :c:func:`PyType_GetModuleState_DuringGC`, + :c:func:`PyModule_GetState_DuringGC`, :c:func:`PyModule_GetToken_DuringGC`, + :c:func:`PyType_GetBaseByToken_DuringGC`, + :c:func:`PyType_GetModule_DuringGC`, + :c:func:`PyType_GetModuleByToken_DuringGC`. + (Contributed by Petr Viktorin in :gh:`145925`.) + +* Add :c:func:`PyObject_Dump` to dump an object to ``stderr``. + It should only be used for debugging. + (Contributed by Victor Stinner in :gh:`141070`.) + +* Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and + :c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set + the stack protection base address and stack protection size of a Python + thread state. + (Contributed by Victor Stinner in :gh:`139653`.) + +* Add :c:func:`PyUnstable_SetImmortal` C-API function to mark objects as :term:`immortal`. + (Contributed by Kumar Aditya in :gh:`143300`.) + +* Restore private provisional ``_Py_InitializeMain()`` function removed in + Python 3.14. + (Contributed by Victor Stinner in :gh:`142417`.) + +Changed C APIs +-------------- + +* If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` or :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` + flag is set then :c:macro:`Py_TPFLAGS_HAVE_GC` must be set too. + (Contributed by Sergey Miryanov in :gh:`134786`.) + +* :c:macro:`PyDateTime_IMPORT` is now thread safe. Code that directly checks ``PyDateTimeAPI`` + for ``NULL`` should be updated to call :c:macro:`PyDateTime_IMPORT` instead. + (Contributed by Kumar Aditya in :gh:`141563`.) Porting to Python 3.15 ---------------------- -* :class:`sqlite3.Connection` APIs has been cleaned up. - - * All parameters of :func:`sqlite3.connect` except *database* are now keyword-only. - * The first three parameters of methods :meth:`~sqlite3.Connection.create_function` - and :meth:`~sqlite3.Connection.create_aggregate` are now positional-only. - * The first parameter of methods :meth:`~sqlite3.Connection.set_authorizer`, - :meth:`~sqlite3.Connection.set_progress_handler` and - :meth:`~sqlite3.Connection.set_trace_callback` is now positional-only. - - (Contributed by Serhiy Storchaka in :gh:`133595`.) - * Private functions promoted to public C APIs: The |pythoncapi_compat_project| can be used to get most of these new functions on Python 3.14 and older. -* :data:`resource.RLIM_INFINITY` is now always positive. - Passing a negative integer value that corresponded to its old value - (such as ``-1`` or ``-3``, depending on platform) to - :func:`resource.setrlimit` and :func:`resource.prlimit` is now deprecated. - (Contributed by Serhiy Storchaka in :gh:`137044`.) - -* :meth:`~mmap.mmap.resize` has been removed on platforms that don't support the - underlying syscall, instead of raising a :exc:`SystemError`. - - -Deprecated C APIs ------------------ - -* For unsigned integer formats in :c:func:`PyArg_ParseTuple`, - accepting Python integers with value that is larger than the maximal value - for the C type or less than the minimal value for the corresponding - signed integer type of the same size is now deprecated. - (Contributed by Serhiy Storchaka in :gh:`132629`.) - -* Deprecate :c:member:`~PyComplexObject.cval` field of the the - :c:type:`PyComplexObject` type. - Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex` - to convert a Python complex number to/from the C :c:type:`Py_complex` - representation. - (Contributed by Sergey B Kirpichev in :gh:`128813`.) - -* Functions :c:func:`_Py_c_sum`, :c:func:`_Py_c_diff`, :c:func:`_Py_c_neg`, - :c:func:`_Py_c_prod`, :c:func:`_Py_c_quot`, :c:func:`_Py_c_pow` and - :c:func:`_Py_c_abs` are :term:`soft deprecated`. - (Contributed by Sergey B Kirpichev in :gh:`128813`.) - - -.. Add C API deprecations above alphabetically, not here at the end. Removed C APIs -------------- @@ -777,6 +2127,16 @@ Removed C APIs of :c:func:`PyImport_ImportModule`. (Contributed by Bénédikt Tran in :gh:`133644`.) +* :c:func:`!PyWeakref_GetObject` and :c:macro:`!PyWeakref_GET_OBJECT`: + use :c:func:`PyWeakref_GetRef` instead. The |pythoncapi_compat_project| + can be used to get :c:func:`!PyWeakref_GetRef` on Python 3.12 and older. + (Contributed by Bénédikt Tran in :gh:`133644`.) + +* Remove deprecated :c:func:`!PySys_ResetWarnOptions`. + Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. + + (Contributed by Nikita Sobolev in :gh:`138886`.) + The following functions are removed in favor of :c:func:`PyConfig_Get`. The |pythoncapi_compat_project| can be used to get :c:func:`!PyConfig_Get` on Python 3.13 and older. @@ -813,3 +2173,182 @@ on Python 3.13 and older. .. |pythoncapi_compat_project| replace:: |pythoncapi_compat_project_link|_ .. |pythoncapi_compat_project_link| replace:: pythoncapi-compat project .. _pythoncapi_compat_project_link: https://github.com/python/pythoncapi-compat + + +Deprecated C APIs +----------------- + +* Deprecate :pep:`456` support for providing an external definition + of the string hashing scheme. Removal is scheduled for Python 3.19. + + Previously, embedders could define :c:macro:`Py_HASH_ALGORITHM` to be + ``Py_HASH_EXTERNAL`` to indicate that the hashing scheme was provided + externally but this feature was undocumented, untested and most likely + unused. + + (Contributed by Bénédikt Tran in :gh:`141226`.) + +* For unsigned integer formats in :c:func:`PyArg_ParseTuple`, + accepting Python integers with value that is larger than the maximal value + for the C type or less than the minimal value for the corresponding + signed integer type of the same size is now deprecated. + (Contributed by Serhiy Storchaka in :gh:`132629`.) + +* :c:func:`PyBytes_FromStringAndSize(NULL, len) ` + and :c:func:`_PyBytes_Resize` are :term:`soft deprecated`, + use the :c:type:`PyBytesWriter` API instead. + (Contributed by Victor Stinner in :gh:`129813`.) + +* :c:func:`!_PyObject_CallMethodId`, :c:func:`!_PyObject_GetAttrId` and + :c:func:`!_PyUnicode_FromId` are deprecated since 3.15 and will be removed in + 3.20. Instead, use :c:func:`PyUnicode_InternFromString()` and cache the result in + the module state, then call :c:func:`PyObject_CallMethod` or + :c:func:`PyObject_GetAttr`. + (Contributed by Victor Stinner in :gh:`141049`.) + +* Deprecate :c:member:`~PyComplexObject.cval` field of the + :c:type:`PyComplexObject` type. + Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex` + to convert a Python complex number to/from the C :c:type:`Py_complex` + representation. + (Contributed by Sergey B Kirpichev in :gh:`128813`.) + +* Functions :c:func:`_Py_c_sum`, :c:func:`_Py_c_diff`, :c:func:`_Py_c_neg`, + :c:func:`_Py_c_prod`, :c:func:`_Py_c_quot`, :c:func:`_Py_c_pow` and + :c:func:`_Py_c_abs` are :term:`soft deprecated`. + (Contributed by Sergey B Kirpichev in :gh:`128813`.) + +* :c:member:`~PyConfig.bytes_warning` is deprecated + since 3.15 and will be removed in 3.17. + (Contributed by Nikita Sobolev in :gh:`136355`.) + +* :c:macro:`!Py_INFINITY` macro is :term:`soft deprecated`, + use the C11 standard ```` :c:macro:`!INFINITY` instead. + (Contributed by Sergey B Kirpichev in :gh:`141004`.) + +* The following macros are :term:`soft deprecated`: + + - :c:macro:`Py_ALIGNED`: Prefer ``alignas`` instead. + - :c:macro:`PY_FORMAT_SIZE_T`: Use ``"z"`` directly. + - :c:macro:`Py_LL` and :c:macro:`Py_ULL`: + Use standard suffixes, ``LL`` and ``ULL``. + - :c:macro:`PY_LONG_LONG`, :c:macro:`PY_LLONG_MIN`, :c:macro:`PY_LLONG_MAX`, + :c:macro:`PY_ULLONG_MAX`, :c:macro:`PY_INT32_T`, :c:macro:`PY_UINT32_T`, + :c:macro:`PY_INT64_T`, :c:macro:`PY_UINT64_T`, :c:macro:`PY_SIZE_MAX`: + Use C99 types/limits. + - :c:macro:`Py_UNICODE_SIZE`: Use ``sizeof(wchar_t)`` directly. + - :c:macro:`Py_VA_COPY`: Use ``va_copy`` directly. + + The macro :c:macro:`Py_UNICODE_WIDE`, which was scheduled for removal, + is :term:`soft deprecated` instead. + + (Contributed by Petr Viktorin in :gh:`146175`.) + +* :c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated + since 3.15 and will be removed in 3.20. + (Contributed by Sergey B Kirpichev in :gh:`141004`.) + + +.. Add C API deprecations above alphabetically, not here at the end. + + +Build changes +============= + +* Removed implicit fallback to the bundled copy of the ``libmpdec`` library. + Now this should be explicitly enabled with :option:`--with-system-libmpdec` + set to ``no`` or with :option:`!--without-system-libmpdec`. + (Contributed by Sergey B Kirpichev in :gh:`115119`.) + +* The new configure option :option:`--with-missing-stdlib-config=FILE` allows + distributors to pass a `JSON `_ + configuration file containing custom error messages for :term:`standard library` + modules that are missing or packaged separately. + (Contributed by Stan Ulbrych and Petr Viktorin in :gh:`139707`.) + +* The new configure option :option:`--with-pymalloc-hugepages` enables huge + page support for :ref:`pymalloc ` arenas. When enabled, arena size + increases to 2 MiB and allocation uses ``MAP_HUGETLB`` (Linux) or + ``MEM_LARGE_PAGES`` (Windows) with automatic fallback to regular pages. + On Windows, use ``build.bat --pymalloc-hugepages``. + At runtime, huge pages must be explicitly enabled by setting the + :envvar:`PYTHON_PYMALLOC_HUGEPAGES` environment variable to ``1``. + +* Annotating anonymous mmap usage is now supported if Linux kernel supports + :manpage:`PR_SET_VMA_ANON_NAME ` (Linux 5.17 or newer). + Annotations are visible in ``/proc//maps`` if the kernel supports the feature + and :option:`-X dev <-X>` is passed to the Python or Python is built in :ref:`debug mode `. + (Contributed by Donghee Na in :gh:`141770`.) + +.. _whatsnew315-frame-pointers: + +* CPython is now built with frame pointers enabled by default + (:pep:`831`). Pass :option:`--without-frame-pointers` to opt out. + Authors of C extensions and native libraries built with custom build + systems should add ``-fno-omit-frame-pointer`` and + ``-mno-omit-leaf-frame-pointer`` to their own ``CFLAGS`` to keep the + unwind chain intact. + (Contributed by Pablo Galindo Salgado and Savannah Ostrowski in :gh:`149201`.) + +.. _whatsnew315-windows-tail-calling-interpreter: + +* 64-bit builds using Visual Studio 2026 (MSVC 18) may now use the new + :ref:`tail-calling interpreter `. + Results on Visual Studio 18.1.1 report between + `15-20% `__ + speedup on the geometric mean of pyperformance on Windows x86-64 over + the switch-case interpreter on an AMD Ryzen 7 5800X. We have + observed speedups ranging from 14% for large pure-Python libraries + to 40% for long-running small pure-Python scripts on Windows. + This was made possible by a new feature introduced in MSVC 18, + which the official Windows 64-bit binaries on python.org__ now use. + (Contributed by Chris Eibl, Ken Jin, and Brandt Bucher in :gh:`143068`. + Special thanks to Steve Dower, and the MSVC team including Hulon Jenkins.) + + __ https://www.python.org/downloads/windows/ + + +Porting to Python 3.15 +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + +* :class:`sqlite3.Connection` APIs have been cleaned up. + + * All parameters of :func:`sqlite3.connect` except *database* are now keyword-only. + * The first three parameters of methods :meth:`~sqlite3.Connection.create_function` + and :meth:`~sqlite3.Connection.create_aggregate` are now positional-only. + * The first parameter of methods :meth:`~sqlite3.Connection.set_authorizer`, + :meth:`~sqlite3.Connection.set_progress_handler` and + :meth:`~sqlite3.Connection.set_trace_callback` is now positional-only. + + (Contributed by Serhiy Storchaka in :gh:`133595`.) + +* :data:`resource.RLIM_INFINITY` is now always positive. + Passing a negative integer value that corresponded to its old value + (such as ``-1`` or ``-3``, depending on platform) to + :func:`resource.setrlimit` and :func:`resource.prlimit` is now deprecated. + (Contributed by Serhiy Storchaka in :gh:`137044`.) + +* :meth:`mmap.mmap.resize` has been removed on platforms that don't support the + underlying syscall, instead of raising a :exc:`SystemError`. + +* A resource warning is now emitted for an unclosed + :func:`xml.etree.ElementTree.iterparse` iterator if it opened a file. + Use its :meth:`!close` method or the :func:`contextlib.closing` context + manager to close it. + (Contributed by Osama Abdelkader and Serhiy Storchaka in :gh:`140601`.) + +* If a short option and a single-dash long option are passed to + :meth:`argparse.ArgumentParser.add_argument`, *dest* is now inferred from + the single-dash long option. For example, in ``add_argument('-f', '-foo')``, + *dest* is now ``'foo'`` instead of ``'f'``. + Pass an explicit *dest* argument to preserve the old behavior. + (Contributed by Serhiy Storchaka in :gh:`138697`.) + +* Padding of input no longer required in :func:`base64.urlsafe_b64decode`. + Pass a new argument ``padded=True`` or use :func:`base64.b64decode` + with argument ``altchars=b'-_'`` (this works with older Python versions) + to make padding required. + (Contributed by Serhiy Storchaka in :gh:`73613`.) diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 47c4d9acbc8..48c461d891e 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -312,7 +312,7 @@ cluttering source directories, the *pyc* files are now collected in a Aside from the filenames and target directories, the new scheme has a few aspects that are visible to the programmer: -* Imported modules now have a :attr:`~module.__cached__` attribute which stores +* Imported modules now have a ``__cached__`` attribute which stores the name of the actual file that was imported: >>> import collections @@ -992,12 +992,12 @@ datetime and time offset and timezone name. This makes it easier to create timezone-aware datetime objects:: - >>> from datetime import datetime, timezone + >>> import datetime as dt - >>> datetime.now(timezone.utc) + >>> dt.datetime.now(dt.timezone.utc) datetime.datetime(2010, 12, 8, 21, 4, 2, 923754, tzinfo=datetime.timezone.utc) - >>> datetime.strptime("01/01/2000 12:00 +0000", "%m/%d/%Y %H:%M %z") + >>> dt.datetime.strptime("01/01/2000 12:00 +0000", "%m/%d/%Y %H:%M %z") datetime.datetime(2000, 1, 1, 12, 0, tzinfo=datetime.timezone.utc) * Also, :class:`~datetime.timedelta` objects can now be multiplied by diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 59afd6520c4..a390211ddb5 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -2,7 +2,7 @@ What's New In Python 3.4 **************************** -:Author: R. David Murray (Editor) +:Author: \R. David Murray (Editor) .. Rules for maintenance: @@ -2086,10 +2086,10 @@ Deprecations in the Python API :meth:`!importlib.abc.PathEntryFinder.find_loader` and :meth:`!find_module` are replaced by :meth:`importlib.abc.PathEntryFinder.find_spec`; all of the :samp:`{xxx}Loader` ABC - ``load_module`` methods (:meth:`!importlib.abc.Loader.load_module`, - :meth:`!importlib.abc.InspectLoader.load_module`, - :meth:`!importlib.abc.FileLoader.load_module`, - :meth:`!importlib.abc.SourceLoader.load_module`) should no longer be + ``load_module`` methods (``importlib.abc.Loader.load_module``, + ``importlib.abc.InspectLoader.load_module``, + ``importlib.abc.FileLoader.load_module``, + ``importlib.abc.SourceLoader.load_module``) should no longer be implemented, instead loaders should implement an ``exec_module`` method (:meth:`importlib.abc.Loader.exec_module`, diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index 59e8ecebb10..6009dd8a71e 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -936,7 +936,7 @@ methods to match the corresponding methods of :class:`str`. collections.abc --------------- -The :meth:`Sequence.index() ` method now +The :meth:`!Sequence.index` method now accepts *start* and *stop* arguments to match the corresponding methods of :class:`tuple`, :class:`list`, etc. (Contributed by Devin Jeanpierre in :issue:`23086`.) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b1e3269239d..bdd35d39e36 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2006,10 +2006,10 @@ deprecated. importlib ~~~~~~~~~ -The :meth:`importlib.machinery.SourceFileLoader.load_module` and -:meth:`importlib.machinery.SourcelessFileLoader.load_module` methods +The ``importlib.machinery.SourceFileLoader.load_module`` and +``importlib.machinery.SourcelessFileLoader.load_module`` methods are now deprecated. They were the only remaining implementations of -:meth:`importlib.abc.Loader.load_module` in :mod:`importlib` that had not +``importlib.abc.Loader.load_module`` in :mod:`importlib` that had not been deprecated in previous versions of Python in favour of :meth:`importlib.abc.Loader.exec_module`. @@ -2173,7 +2173,7 @@ Changes in the Python API * :c:func:`PyErr_SetImportError` now sets :exc:`TypeError` when its **msg** argument is not set. Previously only ``NULL`` was returned. -* The format of the :attr:`~codeobject.co_lnotab` attribute of code objects +* The format of the :attr:`!codeobject.co_lnotab` attribute of code objects changed to support a negative line number delta. By default, Python does not emit bytecode with a negative line number delta. Functions using :attr:`frame.f_lineno`, diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 6e6934befcc..5dd47cdac96 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -604,7 +604,7 @@ new ABC for access to, opening, and reading *resources* inside packages. Resources are roughly similar to files inside packages, but they needn't be actual files on the physical file system. Module loaders can provide a :meth:`!get_resource_reader` function which returns -a :class:`importlib.abc.ResourceReader` instance to support this +a :class:`!importlib.abc.ResourceReader` instance to support this new API. Built-in file path loaders and zip file loaders both support this. Contributed by Barry Warsaw and Brett Cannon in :issue:`32248`. @@ -841,7 +841,7 @@ and by Alexander Mohr and Ilya Kulakov in :issue:`29302`.) cProfile -------- -The :mod:`cProfile` command line now accepts ``-m module_name`` as an +The :mod:`!cProfile` command line now accepts ``-m module_name`` as an alternative to script path. (Contributed by Sanyam Khurana in :issue:`21862`.) @@ -1043,7 +1043,7 @@ window are shown and hidden in the Options menu. importlib --------- -The :class:`importlib.abc.ResourceReader` ABC was introduced to +The :class:`!importlib.abc.ResourceReader` ABC was introduced to support the loading of resources from packages. See also :ref:`whatsnew37_importlib_resources`. (Contributed by Barry Warsaw, Brett Cannon in :issue:`32248`.) @@ -2032,7 +2032,7 @@ both deprecated in Python 3.4 now emit :exc:`DeprecationWarning`. (Contributed by Matthias Bussonnier in :issue:`29576`.) The :class:`importlib.abc.ResourceLoader` ABC has been deprecated in -favour of :class:`importlib.abc.ResourceReader`. +favour of :class:`!importlib.abc.ResourceReader`. locale @@ -2483,7 +2483,7 @@ avoiding possible problems use new functions :c:func:`PySlice_Unpack` and CPython bytecode changes ------------------------ -There are two new opcodes: :opcode:`LOAD_METHOD` and :opcode:`!CALL_METHOD`. +There are two new opcodes: :opcode:`!LOAD_METHOD` and :opcode:`!CALL_METHOD`. (Contributed by Yury Selivanov and INADA Naoki in :issue:`26110`.) The :opcode:`!STORE_ANNOTATION` opcode has been removed. diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 545a17aecab..5078fc30ac1 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -50,7 +50,6 @@ For full details, see the :ref:`changelog `. .. testsetup:: - from datetime import date from math import cos, radians from unicodedata import normalize import re @@ -208,14 +207,15 @@ subdirectories). Debug build uses the same ABI as release build ----------------------------------------------- -Python now uses the same ABI whether it's built in release or debug mode. On -Unix, when Python is built in debug mode, it is now possible to load C -extensions built in release mode and C extensions built using the stable ABI. +The ABI of Python :ref:`debug builds ` is now compatible with +Python release builds. On Unix, when Python is built in debug mode, it is now +possible to load C extensions built in release mode and C extensions built +using the stable ABI. The inverse is not true, as debug builds expose +additional symbols not available in release builds. -Release builds and :ref:`debug builds ` are now ABI compatible: defining the -``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro, which -introduces the only ABI incompatibility. The ``Py_TRACE_REFS`` macro, which -adds the :func:`sys.getobjects` function and the :envvar:`PYTHONDUMPREFS` +Defining the ``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro, +which introduces the only ABI incompatibility. The ``Py_TRACE_REFS`` macro, +which adds the :func:`sys.getobjects` function and the :envvar:`PYTHONDUMPREFS` environment variable, can be set using the new :option:`./configure --with-trace-refs <--with-trace-refs>` build option. (Contributed by Victor Stinner in :issue:`36465`.) @@ -259,15 +259,16 @@ Added an ``=`` specifier to :term:`f-string`\s. An f-string such as ``f'{expr=}'`` will expand to the text of the expression, an equal sign, then the representation of the evaluated expression. For example: + >>> import datetime as dt >>> user = 'eric_idle' - >>> member_since = date(1975, 7, 31) + >>> member_since = dt.date(1975, 7, 31) >>> f'{user=} {member_since=}' "user='eric_idle' member_since=datetime.date(1975, 7, 31)" The usual :ref:`f-string format specifiers ` allow more control over how the result of the expression is displayed:: - >>> delta = date.today() - member_since + >>> delta = dt.date.today() - member_since >>> f'{user=!s} {delta.days=:,d}' 'user=eric_idle delta.days=16,075' diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 40d4a27bff9..49a52b7504b 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -282,20 +282,20 @@ the standard library. It adds :class:`zoneinfo.ZoneInfo`, a concrete Example:: >>> from zoneinfo import ZoneInfo - >>> from datetime import datetime, timedelta + >>> import datetime as dt >>> # Daylight saving time - >>> dt = datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) - >>> print(dt) + >>> when = dt.datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) + >>> print(when) 2020-10-31 12:00:00-07:00 - >>> dt.tzname() + >>> when.tzname() 'PDT' >>> # Standard time - >>> dt += timedelta(days=7) - >>> print(dt) + >>> when += dt.timedelta(days=7) + >>> print(when) 2020-11-07 12:00:00-08:00 - >>> print(dt.tzname()) + >>> print(when.tzname()) PST diff --git a/Grammar/python.gram b/Grammar/python.gram index b9ecd2273a5..9bf3a67939f 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -121,9 +121,9 @@ simple_stmts[asdl_stmt_seq*]: simple_stmt[stmt_ty] (memo): | assignment | &"type" type_alias + | &('import' | 'from' | "lazy") import_stmt | e=star_expressions { _PyAST_Expr(e, EXTRA) } | &'return' return_stmt - | &('import' | 'from') import_stmt | &'raise' raise_stmt | &'pass' pass_stmt | &'del' del_stmt @@ -216,7 +216,7 @@ assert_stmt[stmt_ty]: | invalid_assert_stmt | 'assert' a=expression b=[',' z=expression { z }] { _PyAST_Assert(a, b, EXTRA) } -import_stmt[stmt_ty]: +import_stmt[stmt_ty](memo): | invalid_import | import_name | import_from @@ -224,13 +224,15 @@ import_stmt[stmt_ty]: # Import statements # ----------------- -import_name[stmt_ty]: 'import' a=dotted_as_names { _PyAST_Import(a, EXTRA) } +import_name[stmt_ty]: + | lazy="lazy"? 'import' a=dotted_as_names { _PyAST_Import(a, lazy ? 1 : 0, EXTRA) } + # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS import_from[stmt_ty]: - | 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets { - _PyPegen_checked_future_import(p, b->v.Name.id, c, _PyPegen_seq_count_dots(a), EXTRA) } - | 'from' a=('.' | '...')+ 'import' b=import_from_targets { - _PyAST_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), EXTRA) } + | lazy="lazy"? 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets { + _PyPegen_checked_future_import(p, b->v.Name.id, c, _PyPegen_seq_count_dots(a), lazy, EXTRA) } + | lazy="lazy"? 'from' a=('.' | '...')+ 'import' b=import_from_targets { + _PyAST_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), lazy ? 1 : 0, EXTRA) } import_from_targets[asdl_alias_seq*]: | '(' a=import_from_as_names [','] ')' { a } | import_from_as_names !',' @@ -552,10 +554,12 @@ complex_number[expr_ty]: signed_number[expr_ty]: | NUMBER + | '+' number=NUMBER { number } | '-' number=NUMBER { _PyAST_UnaryOp(USub, number, EXTRA) } signed_real_number[expr_ty]: | real_number + | '+' real=real_number { real } | '-' real=real_number { _PyAST_UnaryOp(USub, real, EXTRA) } real_number[expr_ty]: @@ -563,6 +567,7 @@ real_number[expr_ty]: imaginary_number[expr_ty]: | imag=NUMBER { _PyPegen_ensure_imaginary(p, imag) } + | '+' imag=NUMBER { _PyPegen_ensure_imaginary(p, imag) } capture_pattern[pattern_ty]: | target=pattern_capture_target { _PyAST_MatchAs(NULL, target->v.Name.id, EXTRA) } @@ -626,6 +631,7 @@ mapping_pattern[pattern_ty]: CHECK(asdl_pattern_seq*, _PyPegen_get_patterns(p, items)), NULL, EXTRA) } + | invalid_mapping_pattern items_pattern[asdl_seq*]: | ','.key_value_pattern+ @@ -708,12 +714,16 @@ expressions[expr_ty]: | expression expression[expr_ty] (memo): + | invalid_if_expression | invalid_expression | invalid_legacy_expression - | a=disjunction 'if' b=disjunction 'else' c=expression { _PyAST_IfExp(b, a, c, EXTRA) } + | if_expression | disjunction | lambdef +if_expression[expr_ty]: + | a=disjunction 'if' b=disjunction 'else' c=expression { _PyAST_IfExp(b, a, c, EXTRA) } + yield_expr[expr_ty]: | 'yield' 'from' a=expression { _PyAST_YieldFrom(a, EXTRA) } | 'yield' a=[star_expressions] { _PyAST_Yield(a, EXTRA) } @@ -730,10 +740,16 @@ star_expression[expr_ty] (memo): star_named_expressions[asdl_expr_seq*]: a[asdl_expr_seq*]=','.star_named_expression+ [','] { a } +star_named_expressions_sequence[asdl_expr_seq*]: a[asdl_expr_seq*]=','.star_named_expression_sequence+ [','] { a } + star_named_expression[expr_ty]: | '*' a=bitwise_or { _PyAST_Starred(a, Load, EXTRA) } | named_expression +star_named_expression_sequence[expr_ty]: + | invalid_starred_expression_unpacking_sequence + | star_named_expression + assignment_expression[expr_ty]: | a=NAME ':=' ~ b=expression { CHECK_VERSION(expr_ty, 8, "Assignment expressions are", @@ -881,9 +897,9 @@ atom[expr_ty]: | 'None' { _PyAST_Constant(Py_None, NULL, EXTRA) } | &(STRING|FSTRING_START|TSTRING_START) strings | NUMBER - | &'(' (tuple | group | genexp) - | &'[' (list | listcomp) - | &'{' (dict | set | dictcomp | setcomp) + | &'(' (genexp | tuple | group) + | &'[' (listcomp | list) + | &'{' (dictcomp | setcomp | dict | set) | '...' { _PyAST_Constant(Py_Ellipsis, NULL, EXTRA) } group[expr_ty]: @@ -997,13 +1013,13 @@ strings[expr_ty] (memo): | a[asdl_expr_seq*]=tstring+ { _PyPegen_concatenate_tstrings(p, a, EXTRA) } list[expr_ty]: - | '[' a=[star_named_expressions] ']' { _PyAST_List(a, Load, EXTRA) } + | '[' a=[star_named_expressions_sequence] ']' { _PyAST_List(a, Load, EXTRA) } tuple[expr_ty]: - | '(' a=[y=star_named_expression ',' z=[star_named_expressions] { _PyPegen_seq_insert_in_front(p, y, z) } ] ')' { + | '(' a=[y=star_named_expression_sequence ',' z=[star_named_expressions_sequence] { _PyPegen_seq_insert_in_front(p, y, z) } ] ')' { _PyAST_Tuple(a, Load, EXTRA) } -set[expr_ty]: '{' a=star_named_expressions '}' { _PyAST_Set(a, EXTRA) } +set[expr_ty]: '{' a=star_named_expressions_sequence '}' { _PyAST_Set(a, EXTRA) } # Dicts # ----- @@ -1039,20 +1055,20 @@ for_if_clause[comprehension_ty]: | invalid_for_target listcomp[expr_ty]: - | '[' a=named_expression b=for_if_clauses ']' { _PyAST_ListComp(a, b, EXTRA) } + | '[' a=star_named_expression b=for_if_clauses ']' { _PyAST_ListComp(a, b, EXTRA) } | invalid_comprehension setcomp[expr_ty]: - | '{' a=named_expression b=for_if_clauses '}' { _PyAST_SetComp(a, b, EXTRA) } + | '{' a=star_named_expression b=for_if_clauses '}' { _PyAST_SetComp(a, b, EXTRA) } | invalid_comprehension genexp[expr_ty]: - | '(' a=( assignment_expression | expression !':=') b=for_if_clauses ')' { _PyAST_GeneratorExp(a, b, EXTRA) } + | '(' a=( assignment_expression | expression !':=' | starred_expression ) b=for_if_clauses ')' { _PyAST_GeneratorExp(a, b, EXTRA) } | invalid_comprehension dictcomp[expr_ty]: | '{' a=kvpair b=for_if_clauses '}' { _PyAST_DictComp(a->key, a->value, b, EXTRA) } - | invalid_dict_comprehension + | '{' '**' a=expression b=for_if_clauses '}' { _PyAST_DictComp(a, NULL, b, EXTRA) } # FUNCTION CALL ARGUMENTS # ======================= @@ -1250,8 +1266,7 @@ invalid_expression: # !(NAME STRING) is not matched so we don't show this error with some invalid string prefixes like: kf"dsfsdf" # Soft keywords need to also be ignored because they can be parsed as NAME NAME | !(NAME STRING | SOFT_KEYWORD) a=disjunction b=expression_without_invalid { - _PyPegen_check_legacy_stmt(p, a) ? NULL : p->tokens[p->mark-1]->level == 0 ? NULL : - RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Perhaps you forgot a comma?") } + _PyPegen_raise_error_for_missing_comma(p, a, b) } | a=disjunction 'if' b=disjunction !('else'|':') { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "expected 'else' after 'if' expression") } | a=disjunction 'if' b=disjunction 'else' !expression { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("expected expression after 'else', but statement is given") } @@ -1262,6 +1277,12 @@ invalid_expression: | a='lambda' [lambda_params] b=':' &TSTRING_MIDDLE { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "t-string: lambda expressions are not allowed without parentheses") } +invalid_if_expression: + | disjunction 'if' b=disjunction 'else' a='*' { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot unpack only part of a conditional expression") } + | disjunction 'if' b=disjunction 'else' a='**' { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use dict unpacking on only part of a conditional expression") } + invalid_named_expression(memo): | a=expression ':=' expression { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( @@ -1326,16 +1347,15 @@ invalid_assert_stmt: invalid_block: | NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block") } invalid_comprehension: - | ('[' | '(' | '{') a=starred_expression for_if_clauses { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "iterable unpacking cannot be used in comprehension") } + | '[' a='**' b=expression for_if_clauses { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in list comprehension") } + | '(' a='**' b=expression for_if_clauses { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in generator expression") } | ('[' | '{') a=star_named_expression ',' b=star_named_expressions for_if_clauses { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, PyPegen_last_item(b, expr_ty), "did you forget parentheses around the comprehension target?") } | ('[' | '{') a=star_named_expression b=',' for_if_clauses { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "did you forget parentheses around the comprehension target?") } -invalid_dict_comprehension: - | '{' a='**' bitwise_or for_if_clauses '}' { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "dict unpacking cannot be used in dict comprehension") } invalid_parameters: | a="/" ',' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "at least one parameter must precede /") } @@ -1415,11 +1435,11 @@ invalid_import: | 'import' token=NEWLINE { RAISE_SYNTAX_ERROR_STARTING_FROM(token, "Expected one or more names after 'import'") } invalid_dotted_as_name: - | dotted_name 'as' !(NAME (',' | ')' | NEWLINE)) a=expression { + | dotted_name 'as' !(NAME (',' | ')' | ';' | NEWLINE)) a=expression { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use %s as import target", _PyPegen_get_expr_name(a)) } invalid_import_from_as_name: - | NAME 'as' !(NAME (',' | ')' | NEWLINE)) a=expression { + | NAME 'as' !(NAME (',' | ')' | ';' | NEWLINE)) a=expression { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use %s as import target", _PyPegen_get_expr_name(a)) } @@ -1430,6 +1450,8 @@ invalid_import_from_targets: RAISE_SYNTAX_ERROR_STARTING_FROM(token, "Expected one or more names after 'import'") } invalid_with_stmt: + | ['async'] 'with' ','.(expression ['as' star_target])+ trailing=',' ':' { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(trailing, "the last 'with' item has a trailing comma") } | ['async'] 'with' ','.(expression ['as' star_target])+ NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | ['async'] 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } invalid_with_stmt_indent: @@ -1476,6 +1498,10 @@ invalid_match_stmt: | "match" subject_expr NEWLINE { CHECK_VERSION(void*, 10, "Pattern matching is", RAISE_SYNTAX_ERROR("expected ':'") ) } | a="match" subject=subject_expr ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after 'match' statement on line %d", a->lineno) } + | a="case" patterns guard? b=':' block { + RAISE_SYNTAX_ERROR_KNOWN_RANGE( + a, b, + "case statement must be inside match statement") } invalid_case_block: | "case" patterns guard? NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | a="case" patterns guard? ':' NEWLINE !INDENT { @@ -1490,6 +1516,10 @@ invalid_class_pattern: PyPegen_first_item(a, pattern_ty), PyPegen_last_item(a, pattern_ty), "positional patterns follow keyword patterns") } +invalid_mapping_pattern: + | '{' (items_pattern ',')? rest=double_star_pattern ',' items_pattern ','? '}' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + rest, + "double star pattern must be the last (right-most) subpattern in the mapping pattern") } invalid_class_argument_pattern[asdl_pattern_seq*]: | [positional_patterns ','] keyword_patterns ',' a=positional_patterns { a } invalid_if_stmt: @@ -1522,19 +1552,32 @@ invalid_class_def_raw: RAISE_INDENTATION_ERROR("expected an indented block after class definition on line %d", a->lineno) } invalid_double_starred_kvpairs: - | ','.double_starred_kvpair+ ',' invalid_kvpair - | expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use a starred expression in a dictionary value") } + | invalid_kvpair_unpacking [','] + | ','.double_starred_kvpair+ ',' (invalid_kvpair | invalid_kvpair_unpacking) | expression a=':' &('}'|',') { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") } +invalid_kvpair_unpacking: + | a='**' b=if_expression { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid double starred expression. Did you forget to wrap the conditional expression in parentheses?") } + | a='*' b=bitwise_or ':' expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use a starred expression in a dictionary key") } + | a='**' b=bitwise_or ':' expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in a dictionary key") } + | expression ':' a='*' b=bitwise_or { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use a starred expression in a dictionary value") } + | expression ':' a='**' b=bitwise_or { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in a dictionary value") } invalid_kvpair: | a=expression !(':') { RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, a->lineno, a->end_col_offset - 1, a->end_lineno, -1, "':' expected after dictionary key") } | expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use a starred expression in a dictionary value") } + | expression ':' a='**' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use dict unpacking in a dictionary value") } | expression a=':' &('}'|',') {RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") } invalid_starred_expression_unpacking: + | a='*' b=if_expression { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid starred expression. Did you forget to wrap the conditional expression in parentheses?") } | a='*' expression '=' b=expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to iterable argument unpacking") } +invalid_starred_expression_unpacking_sequence: + | a='**' bitwise_or { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use dict unpacking here") } + | invalid_starred_expression_unpacking invalid_starred_expression: | '*' { RAISE_SYNTAX_ERROR("Invalid star expression") } - invalid_fstring_replacement_field: | '{' a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '='") } | '{' a='!' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '!'") } diff --git a/Include/Python.h b/Include/Python.h index 261b4d316bd..d5e38b8b020 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -9,10 +9,11 @@ // is not needed. -// Include Python header files -#include "patchlevel.h" -#include "pyconfig.h" -#include "pymacconfig.h" +// Include Python configuration headers +#include "patchlevel.h" // the Python version +#include "pyconfig.h" // information from configure +#include "pymacconfig.h" // overrides for pyconfig +#include "pyabi.h" // feature/ABI selection // Include standard header files @@ -22,12 +23,13 @@ #include // INT_MAX #include // HUGE_VAL #include // va_list +#include // memcpy() #include // wchar_t #ifdef HAVE_SYS_TYPES_H # include // ssize_t #endif -// , , and headers are no longer used +// , and headers are no longer used // by Python, but kept for the backward compatibility of existing third party C // extensions. They are not included by limited C API version 3.11 and newer. // @@ -37,7 +39,6 @@ # include // errno # include // FILE* # include // getenv() -# include // memcpy() #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030d0000 # include // tolower() @@ -46,17 +47,11 @@ # endif #endif -#if defined(Py_GIL_DISABLED) -# if defined(Py_LIMITED_API) && !defined(_Py_OPAQUE_PYOBJECT) -# error "Py_LIMITED_API is not currently supported in the free-threaded build" -# endif - -# if defined(_MSC_VER) -# include // __readgsqword() -# endif - -# if defined(__MINGW32__) -# include // __readgsqword() +#if !defined(Py_LIMITED_API) +# if defined(Py_GIL_DISABLED) +# if defined(_MSC_VER) || defined(__MINGW32__) +# include // __readgsqword() +# endif # endif #endif // Py_GIL_DISABLED @@ -71,6 +66,7 @@ __pragma(warning(disable: 4201)) // Include Python header files #include "pyport.h" +#include "exports.h" #include "pymacro.h" #include "pymath.h" #include "pymem.h" @@ -78,7 +74,7 @@ __pragma(warning(disable: 4201)) #include "pybuffer.h" #include "pystats.h" #include "pyatomic.h" -#include "pylock.h" +#include "cpython/pylock.h" #include "critical_section.h" #include "object.h" #include "refcount.h" @@ -105,7 +101,7 @@ __pragma(warning(disable: 4201)) #include "setobject.h" #include "methodobject.h" #include "moduleobject.h" -#include "monitoring.h" +#include "cpython/monitoring.h" #include "cpython/funcobject.h" #include "cpython/classobject.h" #include "fileobject.h" @@ -121,6 +117,7 @@ __pragma(warning(disable: 4201)) #include "cpython/genobject.h" #include "descrobject.h" #include "genericaliasobject.h" +#include "cpython/sentinelobject.h" #include "warnings.h" #include "weakrefobject.h" #include "structseq.h" diff --git a/Include/cpython/abstract.h b/Include/cpython/abstract.h index ffd19ccd350..7490ece52e5 100644 --- a/Include/cpython/abstract.h +++ b/Include/cpython/abstract.h @@ -6,7 +6,7 @@ /* Like PyObject_CallMethod(), but expect a _Py_Identifier* as the method name. */ -PyAPI_FUNC(PyObject*) _PyObject_CallMethodId( +Py_DEPRECATED(3.15) PyAPI_FUNC(PyObject*) _PyObject_CallMethodId( PyObject *obj, _Py_Identifier *name, const char *format, ...); diff --git a/Include/cpython/bytearrayobject.h b/Include/cpython/bytearrayobject.h index 4dddef713ce..1edd0820742 100644 --- a/Include/cpython/bytearrayobject.h +++ b/Include/cpython/bytearrayobject.h @@ -5,25 +5,25 @@ /* Object layout */ typedef struct { PyObject_VAR_HEAD - Py_ssize_t ob_alloc; /* How many bytes allocated in ob_bytes */ + /* How many bytes allocated in ob_bytes + + In the current implementation this is equivalent to Py_SIZE(ob_bytes_object). + The value is always loaded and stored atomically for thread safety. + There are API compatibilty concerns with removing so keeping for now. */ + Py_ssize_t ob_alloc; char *ob_bytes; /* Physical backing buffer */ char *ob_start; /* Logical start inside ob_bytes */ Py_ssize_t ob_exports; /* How many buffer exports */ + PyObject *ob_bytes_object; /* PyBytes for zero-copy bytes conversion */ } PyByteArrayObject; -PyAPI_DATA(char) _PyByteArray_empty_string[]; - /* Macros and static inline functions, trading safety for speed */ #define _PyByteArray_CAST(op) \ (assert(PyByteArray_Check(op)), _Py_CAST(PyByteArrayObject*, op)) static inline char* PyByteArray_AS_STRING(PyObject *op) { - PyByteArrayObject *self = _PyByteArray_CAST(op); - if (Py_SIZE(self)) { - return self->ob_start; - } - return _PyByteArray_empty_string; + return _PyByteArray_CAST(op)->ob_start; } #define PyByteArray_AS_STRING(self) PyByteArray_AS_STRING(_PyObject_CAST(self)) diff --git a/Include/cpython/bytesobject.h b/Include/cpython/bytesobject.h index 71c133f173f..85bc2b827df 100644 --- a/Include/cpython/bytesobject.h +++ b/Include/cpython/bytesobject.h @@ -40,3 +40,46 @@ _PyBytes_Join(PyObject *sep, PyObject *iterable) { return PyBytes_Join(sep, iterable); } + + +// --- PyBytesWriter API ----------------------------------------------------- + +typedef struct PyBytesWriter PyBytesWriter; + +PyAPI_FUNC(PyBytesWriter *) PyBytesWriter_Create( + Py_ssize_t size); +PyAPI_FUNC(void) PyBytesWriter_Discard( + PyBytesWriter *writer); +PyAPI_FUNC(PyObject*) PyBytesWriter_Finish( + PyBytesWriter *writer); +PyAPI_FUNC(PyObject*) PyBytesWriter_FinishWithSize( + PyBytesWriter *writer, + Py_ssize_t size); +PyAPI_FUNC(PyObject*) PyBytesWriter_FinishWithPointer( + PyBytesWriter *writer, + void *buf); + +PyAPI_FUNC(void*) PyBytesWriter_GetData( + PyBytesWriter *writer); +PyAPI_FUNC(Py_ssize_t) PyBytesWriter_GetSize( + PyBytesWriter *writer); + +PyAPI_FUNC(int) PyBytesWriter_WriteBytes( + PyBytesWriter *writer, + const void *bytes, + Py_ssize_t size); +PyAPI_FUNC(int) PyBytesWriter_Format( + PyBytesWriter *writer, + const char *format, + ...); + +PyAPI_FUNC(int) PyBytesWriter_Resize( + PyBytesWriter *writer, + Py_ssize_t size); +PyAPI_FUNC(int) PyBytesWriter_Grow( + PyBytesWriter *writer, + Py_ssize_t size); +PyAPI_FUNC(void*) PyBytesWriter_GrowAndUpdatePointer( + PyBytesWriter *writer, + Py_ssize_t size, + void *buf); diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h index ca8109e3248..5b66fa1040d 100644 --- a/Include/cpython/ceval.h +++ b/Include/cpython/ceval.h @@ -23,6 +23,9 @@ _PyEval_RequestCodeExtraIndex(freefunc f) { PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *); PyAPI_FUNC(int) _PyEval_SliceIndexNotNone(PyObject *, Py_ssize_t *); +PyAPI_FUNC(int) _PyEval_UnpackIndices(PyObject *, PyObject *, + Py_ssize_t, + Py_ssize_t *, Py_ssize_t *); // Trampoline API @@ -35,7 +38,7 @@ typedef struct { PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void); PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry( const void *code_addr, - unsigned int code_size, + size_t code_size, const char *entry_name); PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void); PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename); diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 3f0dce03455..84456a709a6 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -282,15 +282,6 @@ typedef struct _line_offsets { */ PyAPI_FUNC(int) _PyCode_CheckLineNumber(int lasti, PyCodeAddressRange *bounds); -/* Create a comparable key used to compare constants taking in account the - * object type. It is used to make sure types are not coerced (e.g., float and - * complex) _and_ to distinguish 0.0 from -0.0 e.g. on IEEE platforms - * - * Return (type(obj), obj, ...): a tuple with variable size (at least 2 items) - * depending on the type and the value. The type is the first item to not - * compare bytes and str which can raise a BytesWarning exception. */ -PyAPI_FUNC(PyObject*) _PyCode_ConstantKey(PyObject *obj); - PyAPI_FUNC(PyObject*) PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, PyObject *lnotab); diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index df9ec7050fc..5e7811416ab 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -32,6 +32,16 @@ typedef struct { PyDictValues *ma_values; } PyDictObject; +// frozendict +PyAPI_DATA(PyTypeObject) PyFrozenDict_Type; +#define PyFrozenDict_Check(op) PyObject_TypeCheck((op), &PyFrozenDict_Type) +#define PyFrozenDict_CheckExact(op) Py_IS_TYPE((op), &PyFrozenDict_Type) + +#define PyAnyDict_CheckExact(ob) \ + (PyDict_CheckExact(ob) || PyFrozenDict_CheckExact(ob)) +#define PyAnyDict_Check(ob) \ + (PyDict_Check(ob) || PyFrozenDict_Check(ob)) + PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, Py_hash_t hash); // PyDict_GetItemStringRef() can be used instead @@ -39,20 +49,10 @@ Py_DEPRECATED(3.14) PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObje PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); -// Inserts `key` with a value `default_value`, if `key` is not already present -// in the dictionary. If `result` is not NULL, then the value associated -// with `key` is returned in `*result` (either the existing value, or the now -// inserted `default_value`). -// Returns: -// -1 on error -// 0 if `key` was not present and `default_value` was inserted -// 1 if `key` was present and `default_value` was not inserted -PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result); - /* Get the number of items of a dictionary. */ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { PyDictObject *mp; - assert(PyDict_Check(op)); + assert(PyAnyDict_Check(op)); mp = _Py_CAST(PyDictObject*, op); #ifdef Py_GIL_DISABLED return _Py_atomic_load_ssize_relaxed(&mp->ma_used); @@ -103,3 +103,6 @@ PyAPI_FUNC(int) PyDict_ClearWatcher(int watcher_id); // Mark given dictionary as "watched" (callback will be called if it is modified) PyAPI_FUNC(int) PyDict_Watch(int watcher_id, PyObject* dict); PyAPI_FUNC(int) PyDict_Unwatch(int watcher_id, PyObject* dict); + +// Create a frozendict. Create an empty dictionary if iterable is NULL. +PyAPI_FUNC(PyObject*) PyFrozenDict_New(PyObject *iterable); diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index 598cd330bc9..9e1599a7648 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -134,7 +134,8 @@ PyAPI_FUNC(PyObject *) PyStaticMethod_New(PyObject *); V(DESTROY) \ V(MODIFY_CODE) \ V(MODIFY_DEFAULTS) \ - V(MODIFY_KWDEFAULTS) + V(MODIFY_KWDEFAULTS) \ + V(MODIFY_QUALNAME) typedef enum { #define PY_DEF_EVENT(EVENT) PyFunction_EVENT_##EVENT, diff --git a/Include/cpython/import.h b/Include/cpython/import.h index 0ce0b1ee6cc..149a20af8b9 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -10,6 +10,13 @@ struct _inittab { PyAPI_DATA(struct _inittab *) PyImport_Inittab; PyAPI_FUNC(int) PyImport_ExtendInittab(struct _inittab *newtab); +// Custom importers may use this API to initialize statically linked +// extension modules directly from a spec and init function, +// without needing to go through inittab +PyAPI_FUNC(PyObject *) PyImport_CreateModuleFromInitfunc( + PyObject *spec, + PyObject *(*initfunc)(void)); + struct _frozen { const char *name; /* ASCII encoded string */ const unsigned char *code; diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 1c979d91a40..1ccc496c63a 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -149,6 +149,7 @@ typedef struct PyConfig { int dump_refs; wchar_t *dump_refs_file; int malloc_stats; + int pymalloc_hugepages; wchar_t *filesystem_encoding; wchar_t *filesystem_errors; wchar_t *pycache_prefix; @@ -190,6 +191,7 @@ typedef struct PyConfig { int enable_gil; int tlbc_enabled; #endif + int lazy_imports; /* --- Path configuration inputs ------------ */ int pathconfig_warnings; diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index 4b6f97a5e47..998ebe68915 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -133,50 +133,16 @@ _PyLong_CompactValue(const PyLongObject *op) assert(PyType_HasFeature(op->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); assert(PyUnstable_Long_IsCompact(op)); sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); + if (sign == 0) { + // gh-147988: Make sure that the digit is zero. + // It helps detecting the usage of uninitialized digits. + assert(op->long_value.ob_digit[0] == 0); + } return sign * (Py_ssize_t)op->long_value.ob_digit[0]; } #define PyUnstable_Long_CompactValue _PyLong_CompactValue - -/* --- Import/Export API -------------------------------------------------- */ - -typedef struct PyLongLayout { - uint8_t bits_per_digit; - uint8_t digit_size; - int8_t digits_order; - int8_t digit_endianness; -} PyLongLayout; - -PyAPI_FUNC(const PyLongLayout*) PyLong_GetNativeLayout(void); - -typedef struct PyLongExport { - int64_t value; - uint8_t negative; - Py_ssize_t ndigits; - const void *digits; - // Member used internally, must not be used for other purpose. - Py_uintptr_t _reserved; -} PyLongExport; - -PyAPI_FUNC(int) PyLong_Export( - PyObject *obj, - PyLongExport *export_long); -PyAPI_FUNC(void) PyLong_FreeExport( - PyLongExport *export_long); - - -/* --- PyLongWriter API --------------------------------------------------- */ - -typedef struct PyLongWriter PyLongWriter; - -PyAPI_FUNC(PyLongWriter*) PyLongWriter_Create( - int negative, - Py_ssize_t ndigits, - void **digits); -PyAPI_FUNC(PyObject*) PyLongWriter_Finish(PyLongWriter *writer); -PyAPI_FUNC(void) PyLongWriter_Discard(PyLongWriter *writer); - #ifdef __cplusplus } #endif diff --git a/Include/cpython/marshal.h b/Include/cpython/marshal.h new file mode 100644 index 00000000000..159459fcaec --- /dev/null +++ b/Include/cpython/marshal.h @@ -0,0 +1,17 @@ +#ifndef _Py_CPYTHON_MARSHAL_H +# error "this header file must not be included directly" +#endif + +PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, + Py_ssize_t); +PyAPI_FUNC(PyObject *) PyMarshal_WriteObjectToString(PyObject *, int); + +#define Py_MARSHAL_VERSION 6 + +PyAPI_FUNC(long) PyMarshal_ReadLongFromFile(FILE *); +PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *); +PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromFile(FILE *); +PyAPI_FUNC(PyObject *) PyMarshal_ReadLastObjectFromFile(FILE *); + +PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int); +PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int); diff --git a/Include/cpython/modsupport.h b/Include/cpython/modsupport.h index 61344421064..cfeee6e8ab3 100644 --- a/Include/cpython/modsupport.h +++ b/Include/cpython/modsupport.h @@ -2,6 +2,19 @@ # error "this header file must not be included directly" #endif +PyAPI_FUNC(int) PyArg_ParseArray( + PyObject *const *args, + Py_ssize_t nargs, + const char *format, + ...); +PyAPI_FUNC(int) PyArg_ParseArrayAndKeywords( + PyObject *const *args, + Py_ssize_t nargs, + PyObject *kwnames, + const char *format, + const char * const *kwlist, + ...); + // A data structure that can be used to run initialization code once in a // thread-safe manner. The C++11 equivalent is std::call_once. typedef struct { @@ -26,13 +39,7 @@ PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywordsFast(PyObject *, PyObject *, struct _PyArg_Parser *, ...); #ifdef Py_BUILD_CORE -// Internal; defined here to avoid explicitly including pycore_modsupport.h -#define _Py_INTERNAL_ABI_SLOT \ - {Py_mod_abi, (void*) &(PyABIInfo) { \ - .abiinfo_major_version = 1, \ - .abiinfo_minor_version = 0, \ - .flags = PyABIInfo_INTERNAL, \ - .build_version = PY_VERSION_HEX, \ - .abi_version = PY_VERSION_HEX }} \ - /////////////////////////////////////////////////////// +// For internal use in stdlib. Needs C99 compound literals. +// Defined here to avoid every stdlib module including pycore_modsupport.h +#define _Py_ABI_SLOT {Py_mod_abi, (void*) &(PyABIInfo) _PyABIInfo_DEFAULT} #endif diff --git a/Include/cpython/monitoring.h b/Include/cpython/monitoring.h index ce92942404c..c93271f6ca9 100644 --- a/Include/cpython/monitoring.h +++ b/Include/cpython/monitoring.h @@ -1,7 +1,13 @@ -#ifndef Py_CPYTHON_MONITORING_H -# error "this header file must not be included directly" +#ifndef Py_MONITORING_H +#define Py_MONITORING_H +#ifndef Py_LIMITED_API +#ifdef __cplusplus +extern "C" { #endif +// There is currently no limited API for monitoring + + /* Local events. * These require bytecode instrumentation */ @@ -18,9 +24,10 @@ #define PY_MONITORING_EVENT_STOP_ITERATION 10 #define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ - ((ev) < _PY_MONITORING_LOCAL_EVENTS) +((ev) <= PY_MONITORING_EVENT_STOP_ITERATION) -/* Other events, mainly exceptions */ +/* Other events, mainly exceptions. + * These can now be turned on and disabled on a per code object basis. */ #define PY_MONITORING_EVENT_RAISE 11 #define PY_MONITORING_EVENT_EXCEPTION_HANDLED 12 @@ -28,6 +35,9 @@ #define PY_MONITORING_EVENT_PY_THROW 14 #define PY_MONITORING_EVENT_RERAISE 15 +#define _PY_MONITORING_IS_UNGROUPED_EVENT(ev) \ +((ev) < _PY_MONITORING_UNGROUPED_EVENTS) + /* Ancillary events */ @@ -267,3 +277,9 @@ PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike } #undef _PYMONITORING_IF_ACTIVE + +#ifdef __cplusplus +} +#endif +#endif // !Py_LIMITED_API +#endif // !Py_MONITORING_H diff --git a/Include/cpython/object.h b/Include/cpython/object.h index b244c062c76..0b50d2a9dd8 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -230,6 +230,8 @@ struct _typeobject { destructor tp_finalize; vectorcallfunc tp_vectorcall; + /* Below here all fields are internal to the VM */ + /* bitset of which type-watchers care about this type */ unsigned char tp_watched; @@ -239,6 +241,7 @@ struct _typeobject { * Otherwise, limited to MAX_VERSIONS_PER_CLASS (defined elsewhere). */ uint16_t tp_versions_used; + _Py_iteritemfunc _tp_iteritem; /* Virtual iterator next function */ }; #define _Py_ATTR_CACHE_UNUSED (30000) // (see tp_versions_used) @@ -295,9 +298,12 @@ PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *); PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); PyAPI_FUNC(void) _Py_BreakPoint(void); -PyAPI_FUNC(void) _PyObject_Dump(PyObject *); +PyAPI_FUNC(void) PyObject_Dump(PyObject *); -PyAPI_FUNC(PyObject*) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); +// Alias for backward compatibility +#define _PyObject_Dump PyObject_Dump + +Py_DEPRECATED(3.15) PyAPI_FUNC(PyObject*) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); PyAPI_FUNC(PyObject **) _PyObject_GetDictPtr(PyObject *); PyAPI_FUNC(void) PyObject_CallFinalizer(PyObject *); @@ -387,10 +393,11 @@ PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); process with a message on stderr if the given condition fails to hold, but compile away to nothing if NDEBUG is defined. - However, before aborting, Python will also try to call _PyObject_Dump() on - the given object. This may be of use when investigating bugs in which a - particular object is corrupt (e.g. buggy a tp_visit method in an extension - module breaking the garbage collector), to help locate the broken objects. + However, before aborting, Python will also try to call + PyObject_Dump() on the given object. This may be of use when + investigating bugs in which a particular object is corrupt (e.g. buggy a + tp_visit method in an extension module breaking the garbage collector), to + help locate the broken objects. The WITH_MSG variant allows you to supply an additional message that Python will attempt to print to stderr, after the object dump. */ @@ -432,17 +439,15 @@ PyAPI_FUNC(void) _Py_NO_RETURN _PyObject_AssertFailed( PyAPI_FUNC(void) _PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op); PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(PyThreadState *tstate); -PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count); - /* For backwards compatibility with the old trashcan mechanism */ #define Py_TRASHCAN_BEGIN(op, dealloc) #define Py_TRASHCAN_END PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj); +PyAPI_FUNC(void *) PyObject_GetItemData_DuringGC(PyObject *obj); PyAPI_FUNC(int) PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg); -PyAPI_FUNC(int) _PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict); PyAPI_FUNC(void) PyObject_ClearManagedDict(PyObject *obj); @@ -463,6 +468,7 @@ PyAPI_FUNC(int) PyUnstable_Type_AssignVersionTag(PyTypeObject *type); typedef enum { PyRefTracer_CREATE = 0, PyRefTracer_DESTROY = 1, + PyRefTracer_TRACKER_REMOVED = 2, } PyRefTracerEvent; typedef int (*PyRefTracer)(PyObject *, PyRefTracerEvent event, void *); @@ -492,6 +498,83 @@ PyAPI_FUNC(void) PyUnstable_EnableTryIncRef(PyObject *); PyAPI_FUNC(int) PyUnstable_Object_IsUniquelyReferenced(PyObject *); -/* Utility for the tp_traverse slot of mutable heap types that have no other - * references. */ -PyAPI_FUNC(int) _PyObject_VisitType(PyObject *op, visitproc visit, void *arg); +PyAPI_FUNC(int) PyUnstable_SetImmortal(PyObject *op); + +#if defined(Py_GIL_DISABLED) +PyAPI_FUNC(uintptr_t) _Py_GetThreadLocal_Addr(void); + +static inline uintptr_t +_Py_ThreadId(void) +{ + uintptr_t tid; +#if defined(_MSC_VER) && defined(_M_X64) + tid = __readgsqword(48); +#elif defined(_MSC_VER) && defined(_M_IX86) + tid = __readfsdword(24); +#elif defined(_MSC_VER) && defined(_M_ARM64) + tid = __getReg(18); +#elif defined(__MINGW32__) && defined(_M_X64) + tid = __readgsqword(48); +#elif defined(__MINGW32__) && defined(_M_IX86) + tid = __readfsdword(24); +#elif defined(__MINGW32__) && defined(_M_ARM64) + tid = __getReg(18); +#elif defined(__i386__) + __asm__("{movl %%gs:0, %0|mov %0, dword ptr gs:[0]}" : "=r" (tid)); // 32-bit always uses GS +#elif defined(__MACH__) && defined(__x86_64__) + __asm__("{movq %%gs:0, %0|mov %0, qword ptr gs:[0]}" : "=r" (tid)); // x86_64 macOSX uses GS +#elif defined(__x86_64__) + __asm__("{movq %%fs:0, %0|mov %0, qword ptr fs:[0]}" : "=r" (tid)); // x86_64 Linux, BSD uses FS +#elif defined(__arm__) && __ARM_ARCH >= 7 + __asm__ ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tid)); +#elif defined(__aarch64__) && defined(__APPLE__) + __asm__ ("mrs %0, tpidrro_el0" : "=r" (tid)); +#elif defined(__aarch64__) + __asm__ ("mrs %0, tpidr_el0" : "=r" (tid)); +#elif defined(__powerpc64__) + #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) + tid = (uintptr_t)__builtin_thread_pointer(); + #else + // r13 is reserved for use as system thread ID by the Power 64-bit ABI. + register uintptr_t tp __asm__ ("r13"); + __asm__("" : "=r" (tp)); + tid = tp; + #endif +#elif defined(__powerpc__) + #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) + tid = (uintptr_t)__builtin_thread_pointer(); + #else + // r2 is reserved for use as system thread ID by the Power 32-bit ABI. + register uintptr_t tp __asm__ ("r2"); + __asm__ ("" : "=r" (tp)); + tid = tp; + #endif +#elif defined(__s390__) && defined(__GNUC__) + // Both GCC and Clang have supported __builtin_thread_pointer + // for s390 from long time ago. + tid = (uintptr_t)__builtin_thread_pointer(); +#elif defined(__riscv) + #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) + tid = (uintptr_t)__builtin_thread_pointer(); + #else + // tp is Thread Pointer provided by the RISC-V ABI. + __asm__ ("mv %0, tp" : "=r" (tid)); + #endif +#else + // Fallback to a portable implementation if we do not have a faster + // platform-specific implementation. + tid = _Py_GetThreadLocal_Addr(); +#endif + return tid; +} + +static inline Py_ALWAYS_INLINE int +_Py_IsOwnedByCurrentThread(PyObject *ob) +{ +#ifdef _Py_THREAD_SANITIZER + return _Py_atomic_load_uintptr_relaxed(&ob->ob_tid) == _Py_ThreadId(); +#else + return ob->ob_tid == _Py_ThreadId(); +#endif +} +#endif diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 2a0c11e7b3a..e85b360c986 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -72,8 +72,8 @@ // def _Py_atomic_load_ptr_acquire(obj): // return obj # acquire // -// def _Py_atomic_store_ptr_release(obj): -// return obj # release +// def _Py_atomic_store_ptr_release(obj, value): +// obj = value # release // // def _Py_atomic_fence_seq_cst(): // # sequential consistency @@ -523,12 +523,18 @@ _Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value); static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); +static inline void +_Py_atomic_store_int8_release(int8_t *obj, int8_t value); + static inline void _Py_atomic_store_int_release(int *obj, int value); static inline int _Py_atomic_load_int_acquire(const int *obj); +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value); + static inline void _Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value); @@ -591,6 +597,17 @@ static inline void _Py_atomic_fence_release(void); // --- aliases --------------------------------------------------------------- +// Compilers don't really support "consume" semantics, so we fake it. Use +// "acquire" with TSan to support false positives. Use "relaxed" otherwise, +// because CPUs on all platforms we support respect address dependencies without +// extra barriers. +// See 2.6.7 in https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2055r0.pdf +#if defined(_Py_THREAD_SANITIZER) +# define _Py_atomic_load_ptr_consume _Py_atomic_load_ptr_acquire +#else +# define _Py_atomic_load_ptr_consume _Py_atomic_load_ptr_relaxed +#endif + #if SIZEOF_LONG == 8 # define _Py_atomic_load_ulong(p) \ _Py_atomic_load_uint64((uint64_t *)p) diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 1566b83b9f6..253b35082aa 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -572,10 +572,18 @@ static inline void _Py_atomic_store_int_release(int *obj, int value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_int8_release(int8_t *obj, int8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline int _Py_atomic_load_int_acquire(const int *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index d155955df0c..3b3c5f7017e 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -971,12 +971,6 @@ _Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) *(volatile unsigned short *)obj = value; } -static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) -{ - *(volatile unsigned int *)obj = value; -} - static inline void _Py_atomic_store_long_relaxed(long *obj, long value) { @@ -1066,6 +1060,32 @@ _Py_atomic_store_int_release(int *obj, int value) #endif } +static inline void +_Py_atomic_store_int8_release(int8_t *obj, int8_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(int8_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int8); + __stlr8((unsigned __int8 volatile *)obj, (unsigned __int8)value); +#else +# error "no implementation of _Py_atomic_store_int8_release" +#endif +} + +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(volatile unsigned int *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_uint_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 69a8b9e615e..faef303da70 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -459,7 +459,7 @@ static inline uint16_t _Py_atomic_load_uint16(const uint16_t *obj) { _Py_USING_STD; - return atomic_load((const _Atomic(uint32_t)*)obj); + return atomic_load((const _Atomic(uint16_t)*)obj); } static inline uint32_t @@ -948,14 +948,6 @@ _Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) memory_order_relaxed); } -static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) -{ - _Py_USING_STD; - atomic_store_explicit((_Atomic(unsigned int)*)obj, value, - memory_order_relaxed); -} - static inline void _Py_atomic_store_long_relaxed(long *obj, long value) { @@ -1031,6 +1023,22 @@ _Py_atomic_store_int_release(int *obj, int value) memory_order_release); } +static inline void +_Py_atomic_store_int8_release(int8_t *obj, int8_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int8_t)*)obj, value, + memory_order_release); +} + +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 6b63d304b0d..be2e3b641c2 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -18,6 +18,7 @@ typedef struct { PyException_HEAD PyObject *msg; PyObject *excs; + PyObject *excs_str; } PyBaseExceptionGroupObject; typedef struct { diff --git a/Include/cpython/pyhash.h b/Include/cpython/pyhash.h index a33ba10b8d3..dac223368db 100644 --- a/Include/cpython/pyhash.h +++ b/Include/cpython/pyhash.h @@ -7,7 +7,7 @@ /* Parameters used for the numeric hash implementation. See notes for _Py_HashDouble in Python/pyhash.c. Numeric hashes are based on - reduction modulo the prime 2**_PyHASH_BITS - 1. */ + reduction modulo the prime 2**PyHASH_BITS - 1. */ #if SIZEOF_VOID_P >= 8 # define PyHASH_BITS 61 @@ -15,7 +15,7 @@ # define PyHASH_BITS 31 #endif -#define PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) +#define PyHASH_MODULUS (((size_t)1 << PyHASH_BITS) - 1) #define PyHASH_INF 314159 #define PyHASH_IMAG PyHASH_MULTIPLIER diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index 86ce6e6f798..e46dfe59ec4 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -25,6 +25,9 @@ PyAPI_FUNC(PyStatus) Py_PreInitializeFromArgs( PyAPI_FUNC(PyStatus) Py_InitializeFromConfig( const PyConfig *config); +// Python 3.8 provisional API (PEP 587) +PyAPI_FUNC(PyStatus) _Py_InitializeMain(void); + PyAPI_FUNC(int) Py_RunMain(void); diff --git a/Include/cpython/pylock.h b/Include/cpython/pylock.h index 63886fca28e..460ac2c9f80 100644 --- a/Include/cpython/pylock.h +++ b/Include/cpython/pylock.h @@ -1,7 +1,11 @@ -#ifndef Py_CPYTHON_LOCK_H -# error "this header file must not be included directly" +#ifndef Py_LOCK_H +#define Py_LOCK_H +#ifndef Py_LIMITED_API +#ifdef __cplusplus +extern "C" { #endif + #define _Py_UNLOCKED 0 #define _Py_LOCKED 1 @@ -72,3 +76,10 @@ _PyMutex_IsLocked(PyMutex *m) return (_Py_atomic_load_uint8(&m->_bits) & _Py_LOCKED) != 0; } #define PyMutex_IsLocked _PyMutex_IsLocked + + +#ifdef __cplusplus +} +#endif +#endif // !Py_LIMITED_API +#endif // !Py_LOCK_H diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 55564194342..0cb57679df3 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -107,11 +107,15 @@ struct _ts { # define _PyThreadState_WHENCE_THREADING 3 # define _PyThreadState_WHENCE_GILSTATE 4 # define _PyThreadState_WHENCE_EXEC 5 +# define _PyThreadState_WHENCE_THREADING_DAEMON 6 #endif /* Currently holds the GIL. Must be its own field to avoid data races */ int holds_gil; + /* Currently requesting the GIL */ + int gil_requested; + int _whence; /* Thread state (_Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, _Py_THREAD_SUSPENDED). @@ -131,6 +135,15 @@ struct _ts { /* Pointer to currently executing frame. */ struct _PyInterpreterFrame *current_frame; + /* Pointer to the base frame (bottommost sentinel frame). + Used by profilers to validate complete stack unwinding. + Points to the embedded base_frame in _PyThreadStateImpl. + The frame is embedded there rather than here because _PyInterpreterFrame + is defined in internal headers that cannot be exposed in the public API. */ + struct _PyInterpreterFrame *base_frame; + + struct _PyInterpreterFrame *last_profiled_frame; + Py_tracefunc c_profilefunc; Py_tracefunc c_tracefunc; PyObject *c_profileobj; @@ -157,6 +170,11 @@ struct _ts { */ unsigned long native_thread_id; + /* List of objects that still need to be cleaned up, singly linked + * via their gc headers' gc_next pointers. The list is populated by + * _PyTrash_thread_deposit_object and cleaned up by + * _PyTrash_thread_destroy_chain. + */ PyObject *delete_later; /* Tagged pointer to top-most critical section, or zero if there is no @@ -180,6 +198,7 @@ struct _ts { _PyStackChunk *datastack_chunk; PyObject **datastack_top; PyObject **datastack_limit; + _PyStackChunk *datastack_cached_chunk; /* XXX signal handlers should also be here */ /* The following fields are here to avoid allocation during init. @@ -211,6 +230,15 @@ struct _ts { */ PyObject *threading_local_sentinel; _PyRemoteDebuggerSupport remote_debugger_support; + +#ifdef Py_STATS + // Pointer to PyStats structure, NULL if recording is off. For the + // free-threaded build, the structure is per-thread (stored as a pointer + // in _PyThreadStateImpl). For the default build, the structure is stored + // in the PyInterpreterState structure (threads do not have their own + // structure and all share the same per-interpreter structure). + PyStats *pystats; +#endif }; /* other API */ @@ -233,6 +261,21 @@ PyAPI_FUNC(void) PyThreadState_EnterTracing(PyThreadState *tstate); // function is set, otherwise disable them. PyAPI_FUNC(void) PyThreadState_LeaveTracing(PyThreadState *tstate); +#ifdef Py_STATS +#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) +extern _Py_thread_local PyThreadState *_Py_tss_tstate; + +static inline PyStats* +_PyThreadState_GetStatsFast(void) +{ + if (_Py_tss_tstate == NULL) { + return NULL; // no attached thread state + } + return _Py_tss_tstate->pystats; +} +#endif +#endif // Py_STATS + /* PyGILState */ /* Helper/diagnostic function - return 1 if the current thread @@ -246,6 +289,18 @@ PyAPI_FUNC(int) PyGILState_Check(void); */ PyAPI_FUNC(PyObject*) _PyThread_CurrentFrames(void); +// Set the stack protection start address and stack protection size +// of a Python thread state +PyAPI_FUNC(int) PyUnstable_ThreadState_SetStackProtection( + PyThreadState *tstate, + void *stack_start_addr, // Stack start address + size_t stack_size); // Stack size (in bytes) + +// Reset the stack protection start address and stack protection size +// of a Python thread state +PyAPI_FUNC(void) PyUnstable_ThreadState_ResetStackProtection( + PyThreadState *tstate); + /* Routines for advanced debuggers, requested by David Beazley. Don't use unless you know what you are doing! */ PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Main(void); @@ -264,3 +319,8 @@ PyAPI_FUNC(_PyFrameEvalFunction) _PyInterpreterState_GetEvalFrameFunc( PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameFunc( PyInterpreterState *interp, _PyFrameEvalFunction eval_frame); +PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameAllowSpecialization( + PyInterpreterState *interp, + int allow_specialization); +PyAPI_FUNC(int) _PyInterpreterState_IsSpecializationEnabled( + PyInterpreterState *interp); diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index cf830b6066f..5d1f44988a6 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -4,7 +4,7 @@ // // - _Py_INCREF_STAT_INC() and _Py_DECREF_STAT_INC() used by Py_INCREF() // and Py_DECREF(). -// - _Py_stats variable +// - _PyStats_GET() // // Functions of the sys module: // @@ -14,7 +14,7 @@ // - sys._stats_dump() // // Python must be built with ./configure --enable-pystats to define the -// Py_STATS macro. +// _PyStats_GET() macro. // // Define _PY_INTERPRETER macro to increment interpreter_increfs and // interpreter_decrefs. Otherwise, increment increfs and decrefs. @@ -29,7 +29,7 @@ # error "this header file must not be included directly" #endif -#define PYSTATS_MAX_UOP_ID 1024 +#define PYSTATS_MAX_UOP_ID 2000 #define SPECIALIZATION_FAILURE_KINDS 60 @@ -109,6 +109,18 @@ typedef struct _gc_stats { uint64_t objects_not_transitively_reachable; } GCStats; +#ifdef Py_GIL_DISABLED +// stats specific to free-threaded build +typedef struct _ft_stats { + // number of times interpreter had to spin or park when trying to acquire a mutex + uint64_t mutex_sleeps; + // number of times that the QSBR mechanism polled (compute read sequence value) + uint64_t qsbr_polls; + // number of times stop-the-world mechanism was used + uint64_t world_stops; +} FTStats; +#endif + typedef struct _uop_stats { uint64_t execution_count; uint64_t miss; @@ -130,7 +142,9 @@ typedef struct _optimization_stats { uint64_t recursive_call; uint64_t low_confidence; uint64_t unknown_callee; + uint64_t trace_immediately_deopts; uint64_t executors_invalidated; + uint64_t fitness_terminated_traces; UOpStats opcode[PYSTATS_MAX_UOP_ID + 1]; uint64_t unsupported_opcode[256]; uint64_t trace_length_hist[_Py_UOP_HIST_SIZE]; @@ -138,6 +152,8 @@ typedef struct _optimization_stats { uint64_t optimized_trace_length_hist[_Py_UOP_HIST_SIZE]; uint64_t optimizer_attempts; uint64_t optimizer_successes; + uint64_t optimizer_contradiction; + uint64_t optimizer_frame_overflow; uint64_t optimizer_failure_reason_no_memory; uint64_t remove_globals_builtins_changed; uint64_t remove_globals_incorrect_keys; @@ -173,22 +189,48 @@ typedef struct _stats { CallStats call_stats; ObjectStats object_stats; OptimizationStats optimization_stats; +#ifdef Py_GIL_DISABLED + FTStats ft_stats; +#endif RareEventStats rare_event_stats; - GCStats *gc_stats; + GCStats gc_stats[3]; // must match NUM_GENERATIONS } PyStats; +// Export for most shared extensions +PyAPI_FUNC(PyStats *) _PyStats_GetLocal(void); -// Export for shared extensions like 'math' -PyAPI_DATA(PyStats*) _Py_stats; +#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) +// use inline function version defined in cpython/pystate.h +static inline PyStats *_PyThreadState_GetStatsFast(void); +#define _PyStats_GET _PyThreadState_GetStatsFast +#else +#define _PyStats_GET _PyStats_GetLocal +#endif + +#define _Py_STATS_EXPR(expr) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (s != NULL) { \ + s->expr; \ + } \ + } while (0) + +#define _Py_STATS_COND_EXPR(cond, expr) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (s != NULL && (cond)) { \ + s->expr; \ + } \ + } while (0) #ifdef _PY_INTERPRETER -# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_increfs++; } while (0) -# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_decrefs++; } while (0) -# define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_increfs++; } while (0) -# define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_decrefs++; } while (0) +# define _Py_INCREF_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_increfs++) +# define _Py_DECREF_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_decrefs++) +# define _Py_INCREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_immortal_increfs++) +# define _Py_DECREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_immortal_decrefs++) #else -# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.increfs++; } while (0) -# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.decrefs++; } while (0) -# define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_increfs++; } while (0) -# define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_decrefs++; } while (0) +# define _Py_INCREF_STAT_INC() _Py_STATS_EXPR(object_stats.increfs++) +# define _Py_DECREF_STAT_INC() _Py_STATS_EXPR(object_stats.decrefs++) +# define _Py_INCREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.immortal_increfs++) +# define _Py_DECREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.immortal_decrefs++) #endif diff --git a/Include/cpython/sentinelobject.h b/Include/cpython/sentinelobject.h new file mode 100644 index 00000000000..0b6ff0f17e6 --- /dev/null +++ b/Include/cpython/sentinelobject.h @@ -0,0 +1,22 @@ +/* Sentinel object interface */ + +#ifndef Py_LIMITED_API +#ifndef Py_SENTINELOBJECT_H +#define Py_SENTINELOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +PyAPI_DATA(PyTypeObject) PySentinel_Type; + +#define PySentinel_Check(op) Py_IS_TYPE((op), &PySentinel_Type) + +PyAPI_FUNC(PyObject *) PySentinel_New( + const char *name, + const char *module_name); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_SENTINELOBJECT_H */ +#endif /* !Py_LIMITED_API */ diff --git a/Include/cpython/sliceobject.h b/Include/cpython/sliceobject.h new file mode 100644 index 00000000000..4c3ea1faceb --- /dev/null +++ b/Include/cpython/sliceobject.h @@ -0,0 +1,20 @@ +#ifndef Py_CPYTHON_SLICEOBJECT_H +# error "this header file must not be included directly" +#endif + +/* Slice object interface */ + +/* +A slice object containing start, stop, and step data members (the +names are from range). After much talk with Guido, it was decided to +let these be any arbitrary python type. Py_None stands for omitted values. +*/ +typedef struct { + PyObject_HEAD + PyObject *start, *stop, *step; /* not NULL */ +} PySliceObject; + +PyAPI_FUNC(PyObject *) _PySlice_FromIndices(Py_ssize_t start, Py_ssize_t stop); +PyAPI_FUNC(int) _PySlice_GetLongIndices(PySliceObject *self, PyObject *length, + PyObject **start_ptr, PyObject **stop_ptr, + PyObject **step_ptr); diff --git a/Include/cpython/structseq.h b/Include/cpython/structseq.h new file mode 100644 index 00000000000..328fbe86143 --- /dev/null +++ b/Include/cpython/structseq.h @@ -0,0 +1,12 @@ +#ifndef Py_CPYTHON_STRUCTSEQ_H +# error "this header file must not be included directly" +#endif + +PyAPI_FUNC(void) PyStructSequence_InitType(PyTypeObject *type, + PyStructSequence_Desc *desc); +PyAPI_FUNC(int) PyStructSequence_InitType2(PyTypeObject *type, + PyStructSequence_Desc *desc); + +typedef PyTupleObject PyStructSequence; +#define PyStructSequence_SET_ITEM PyStructSequence_SetItem +#define PyStructSequence_GET_ITEM PyStructSequence_GetItem diff --git a/Include/cpython/tupleobject.h b/Include/cpython/tupleobject.h index afb98ccbb81..888baaf3358 100644 --- a/Include/cpython/tupleobject.h +++ b/Include/cpython/tupleobject.h @@ -38,3 +38,7 @@ PyTuple_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) { } #define PyTuple_SET_ITEM(op, index, value) \ PyTuple_SET_ITEM(_PyObject_CAST(op), (index), _PyObject_CAST(value)) + +PyAPI_FUNC(PyObject*) PyTuple_FromArray( + PyObject *const *array, + Py_ssize_t size); diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 73e3bc44d6c..ea91f4158eb 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -301,7 +301,6 @@ static inline Py_ssize_t PyUnicode_GET_LENGTH(PyObject *op) { /* Returns the cached hash, or -1 if not cached yet. */ static inline Py_hash_t PyUnstable_Unicode_GET_CACHED_HASH(PyObject *op) { - assert(PyUnicode_Check(op)); #ifdef Py_GIL_DISABLED return _Py_atomic_load_ssize_relaxed(&_PyASCIIObject_CAST(op)->hash); #else @@ -497,7 +496,7 @@ PyAPI_FUNC(int) PyUnicodeWriter_WriteWideChar( Py_ssize_t size); PyAPI_FUNC(int) PyUnicodeWriter_WriteUCS4( PyUnicodeWriter *writer, - Py_UCS4 *str, + const Py_UCS4 *str, Py_ssize_t size); PyAPI_FUNC(int) PyUnicodeWriter_WriteStr( @@ -779,4 +778,4 @@ static inline int Py_UNICODE_ISALNUM(Py_UCS4 ch) { // Return an interned Unicode object for an Identifier; may fail if there is no // memory. -PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*); +Py_DEPRECATED(3.15) PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*); diff --git a/Include/cpython/warnings.h b/Include/cpython/warnings.h index 8731fd2e96b..4e3eb88e8ff 100644 --- a/Include/cpython/warnings.h +++ b/Include/cpython/warnings.h @@ -18,9 +18,3 @@ PyAPI_FUNC(int) PyErr_WarnExplicitFormat( // DEPRECATED: Use PyErr_WarnEx() instead. #define PyErr_Warn(category, msg) PyErr_WarnEx((category), (msg), 1) - -int _PyErr_WarnExplicitObjectWithContext( - PyObject *category, - PyObject *message, - PyObject *filename, - int lineno); diff --git a/Include/cpython/weakrefobject.h b/Include/cpython/weakrefobject.h index da8e77cddac..e0711407cee 100644 --- a/Include/cpython/weakrefobject.h +++ b/Include/cpython/weakrefobject.h @@ -47,20 +47,3 @@ PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); // Test if a weak reference is dead. PyAPI_FUNC(int) PyWeakref_IsDead(PyObject *ref); - -Py_DEPRECATED(3.13) static inline PyObject* PyWeakref_GET_OBJECT(PyObject *ref_obj) -{ - PyWeakReference *ref = _PyWeakref_CAST(ref_obj); - PyObject *obj = ref->wr_object; - // Explanation for the Py_REFCNT() check: when a weakref's target is part - // of a long chain of deallocations which triggers the trashcan mechanism, - // clearing the weakrefs can be delayed long after the target's refcount - // has dropped to zero. In the meantime, code accessing the weakref will - // be able to "see" the target object even though it is supposed to be - // unreachable. See issue gh-60806. - if (Py_REFCNT(obj) > 0) { - return obj; - } - return Py_None; -} -#define PyWeakref_GET_OBJECT(ref) PyWeakref_GET_OBJECT(_PyObject_CAST(ref)) diff --git a/Include/datetime.h b/Include/datetime.h index b78cc0e8e2e..66e6c6e3ac3 100644 --- a/Include/datetime.h +++ b/Include/datetime.h @@ -1,8 +1,8 @@ /* datetime.h */ #ifndef Py_LIMITED_API -#ifndef DATETIME_H -#define DATETIME_H +#ifndef Py_DATETIME_H +#define Py_DATETIME_H #ifdef __cplusplus extern "C" { #endif @@ -196,8 +196,23 @@ typedef struct { /* Define global variable for the C API and a macro for setting it. */ static PyDateTime_CAPI *PyDateTimeAPI = NULL; -#define PyDateTime_IMPORT \ - PyDateTimeAPI = (PyDateTime_CAPI *)PyCapsule_Import(PyDateTime_CAPSULE_NAME, 0) +static inline PyDateTime_CAPI * +_PyDateTime_IMPORT(void) { + PyDateTime_CAPI *val = (PyDateTime_CAPI *)_Py_atomic_load_ptr(&PyDateTimeAPI); + if (val == NULL) { + PyDateTime_CAPI *capi = (PyDateTime_CAPI *)PyCapsule_Import( + PyDateTime_CAPSULE_NAME, 0); + if (capi != NULL) { + /* if the compare exchange fails then in that case + another thread would have initialized it */ + _Py_atomic_compare_exchange_ptr(&PyDateTimeAPI, &val, (void *)capi); + return capi; + } + } + return val; +} + +#define PyDateTime_IMPORT _PyDateTime_IMPORT() /* Macro for access to the UTC singleton */ #define PyDateTime_TimeZone_UTC PyDateTimeAPI->TimeZone_UTC @@ -263,5 +278,5 @@ static PyDateTime_CAPI *PyDateTimeAPI = NULL; #ifdef __cplusplus } #endif -#endif +#endif /* !Py_DATETIME_H */ #endif /* !Py_LIMITED_API */ diff --git a/Include/descrobject.h b/Include/descrobject.h index fd66d17b497..340de4e0e1e 100644 --- a/Include/descrobject.h +++ b/Include/descrobject.h @@ -80,10 +80,14 @@ struct PyMemberDef { #define _Py_T_NONE 20 // Deprecated. Value is always None. /* Flags */ -#define Py_READONLY 1 -#define Py_AUDIT_READ 2 // Added in 3.10, harmless no-op before that -#define _Py_WRITE_RESTRICTED 4 // Deprecated, no-op. Do not reuse the value. -#define Py_RELATIVE_OFFSET 8 +#define Py_READONLY (1 << 0) +#define Py_AUDIT_READ (1 << 1) // Added in 3.10, harmless no-op before that +#define _Py_WRITE_RESTRICTED (1 << 2) // Deprecated, no-op. Do not reuse the value. +#define Py_RELATIVE_OFFSET (1 << 3) + +#ifndef Py_LIMITED_API +# define _Py_AFTER_ITEMS (1 << 4) // For internal use. +#endif PyAPI_FUNC(PyObject *) PyMember_GetOne(const char *, PyMemberDef *); PyAPI_FUNC(int) PyMember_SetOne(char *, PyMemberDef *, PyObject *); diff --git a/Include/dictobject.h b/Include/dictobject.h index 1bbeec1ab69..0384e3131dc 100644 --- a/Include/dictobject.h +++ b/Include/dictobject.h @@ -68,6 +68,18 @@ PyAPI_FUNC(int) PyDict_GetItemRef(PyObject *mp, PyObject *key, PyObject **result PyAPI_FUNC(int) PyDict_GetItemStringRef(PyObject *mp, const char *key, PyObject **result); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030F0000 +// Inserts `key` with a value `default_value`, if `key` is not already present +// in the dictionary. If `result` is not NULL, then the value associated +// with `key` is returned in `*result` (either the existing value, or the now +// inserted `default_value`). +// Returns: +// -1 on error +// 0 if `key` was not present and `default_value` was inserted +// 1 if `key` was present and `default_value` was not inserted +PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result); +#endif + #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 PyAPI_FUNC(PyObject *) PyObject_GenericGetDict(PyObject *, void *); #endif diff --git a/Include/exports.h b/Include/exports.h index 0c646d5beb6..a863ecb3307 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -9,6 +9,7 @@ inside the Python core, they are private to the core. If in an extension module, it may be declared with external linkage depending on the platform. + PyMODEXPORT_FUNC: Like PyMODINIT_FUNC, but for a slots array As a number of platforms support/require "__declspec(dllimport/dllexport)", we support a HAVE_DECLSPEC_DLL macro to save duplication. @@ -34,6 +35,12 @@ #define Py_EXPORTED_SYMBOL #define Py_LOCAL_SYMBOL #endif + /* module init functions outside the core must be exported */ + #if defined(_PyEXPORTS_CORE) + #define _PyINIT_EXPORTED_SYMBOL Py_EXPORTED_SYMBOL + #else + #define _PyINIT_EXPORTED_SYMBOL __declspec(dllexport) + #endif #else /* * If we only ever used gcc >= 5, we could use __has_attribute(visibility) @@ -51,22 +58,19 @@ #define Py_EXPORTED_SYMBOL #define Py_LOCAL_SYMBOL #endif + #define _PyINIT_EXPORTED_SYMBOL Py_EXPORTED_SYMBOL #endif /* only get special linkage if built as shared or platform is Cygwin */ #if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__) # if defined(HAVE_DECLSPEC_DLL) -# if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE -# define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE +# if defined(_PyEXPORTS_CORE) && !defined(_PyEXPORTS_CORE_MODULE) /* module init functions inside the core need no external linkage */ /* except for Cygwin to handle embedding */ -# if defined(__CYGWIN__) -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* -# else /* __CYGWIN__ */ -# define PyMODINIT_FUNC PyObject* +# if !defined(__CYGWIN__) +# define _PyINIT_FUNC_DECLSPEC # endif /* __CYGWIN__ */ -# else /* Py_BUILD_CORE */ +# else /* _PyEXPORTS_CORE */ /* Building an extension module, or an embedded situation */ /* public Python functions and data are imported */ /* Under Cygwin, auto-import functions to prevent compilation */ @@ -76,13 +80,7 @@ # define PyAPI_FUNC(RTYPE) Py_IMPORTED_SYMBOL RTYPE # endif /* !__CYGWIN__ */ # define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE - /* module init functions outside the core must be exported */ -# if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* -# else /* __cplusplus */ -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* -# endif /* __cplusplus */ -# endif /* Py_BUILD_CORE */ +# endif /* _PyEXPORTS_CORE */ # endif /* HAVE_DECLSPEC_DLL */ #endif /* Py_ENABLE_SHARED */ @@ -93,13 +91,19 @@ #ifndef PyAPI_DATA # define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE #endif -#ifndef PyMODINIT_FUNC +#ifndef _PyINIT_FUNC_DECLSPEC # if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC extern "C" _PyINIT_EXPORTED_SYMBOL # else /* __cplusplus */ -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC _PyINIT_EXPORTED_SYMBOL # endif /* __cplusplus */ #endif +#ifndef PyMODINIT_FUNC + #define PyMODINIT_FUNC _PyINIT_FUNC_DECLSPEC PyObject* +#endif +#ifndef PyMODEXPORT_FUNC + #define PyMODEXPORT_FUNC _PyINIT_FUNC_DECLSPEC PyModuleDef_Slot* +#endif #endif /* Py_EXPORTS_H */ diff --git a/Include/floatobject.h b/Include/floatobject.h index 4d24a76edd5..814337b070a 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -18,14 +18,14 @@ PyAPI_DATA(PyTypeObject) PyFloat_Type; #define Py_RETURN_NAN return PyFloat_FromDouble(Py_NAN) -#define Py_RETURN_INF(sign) \ - do { \ - if (copysign(1., sign) == 1.) { \ - return PyFloat_FromDouble(Py_INFINITY); \ - } \ - else { \ - return PyFloat_FromDouble(-Py_INFINITY); \ - } \ +#define Py_RETURN_INF(sign) \ + do { \ + if (copysign(1., sign) == 1.) { \ + return PyFloat_FromDouble(INFINITY); \ + } \ + else { \ + return PyFloat_FromDouble(-INFINITY); \ + } \ } while(0) PyAPI_FUNC(double) PyFloat_GetMax(void); diff --git a/Include/import.h b/Include/import.h index d91ebe96ca8..6f1c13787b8 100644 --- a/Include/import.h +++ b/Include/import.h @@ -88,6 +88,20 @@ PyAPI_FUNC(int) PyImport_AppendInittab( PyObject* (*initfunc)(void) ); +typedef enum { + PyImport_LAZY_NORMAL, + PyImport_LAZY_ALL, + PyImport_LAZY_NONE +} PyImport_LazyImportsMode; + +#ifndef Py_LIMITED_API +PyAPI_FUNC(int) PyImport_SetLazyImportsMode(PyImport_LazyImportsMode mode); +PyAPI_FUNC(int) PyImport_SetLazyImportsFilter(PyObject *filter); + +PyAPI_FUNC(PyImport_LazyImportsMode) PyImport_GetLazyImportsMode(void); +PyAPI_FUNC(PyObject *) PyImport_GetLazyImportsFilter(void); +#endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_IMPORT_H # include "cpython/import.h" diff --git a/Include/internal/mimalloc/mimalloc/types.h b/Include/internal/mimalloc/mimalloc/types.h index 19e93224174..286e7bf6683 100644 --- a/Include/internal/mimalloc/mimalloc/types.h +++ b/Include/internal/mimalloc/mimalloc/types.h @@ -608,8 +608,8 @@ struct mi_heap_s { #if (MI_DEBUG) // use our own assertion to print without memory allocation -mi_decl_noreturn mi_decl_cold mi_decl_throw -void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func); +mi_decl_noreturn mi_decl_cold +void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func) mi_decl_throw; #define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__)) #else #define mi_assert(x) diff --git a/Include/internal/pycore_abstract.h b/Include/internal/pycore_abstract.h index 30809e09700..b9eb4fd9891 100644 --- a/Include/internal/pycore_abstract.h +++ b/Include/internal/pycore_abstract.h @@ -16,10 +16,11 @@ _PyIndex_Check(PyObject *obj) return (tp_as_number != NULL && tp_as_number->nb_index != NULL); } -PyObject *_PyNumber_PowerNoMod(PyObject *lhs, PyObject *rhs); -PyObject *_PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs); +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyNumber_PowerNoMod(PyObject *lhs, PyObject *rhs); +PyAPI_FUNC(PyObject *) _PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs); -extern int _PyObject_HasLen(PyObject *o); +PyAPI_FUNC(int) _PyObject_HasLen(PyObject *o); /* === Sequence protocol ================================================ */ diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index 60367202bab..b47398669bb 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -329,12 +329,14 @@ struct _stmt { struct { asdl_alias_seq *names; + int is_lazy; } Import; struct { identifier module; asdl_alias_seq *names; int level; + int is_lazy; } ImportFrom; struct { @@ -764,11 +766,12 @@ stmt_ty _PyAST_TryStar(asdl_stmt_seq * body, asdl_excepthandler_seq * handlers, end_col_offset, PyArena *arena); stmt_ty _PyAST_Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -stmt_ty _PyAST_Import(asdl_alias_seq * names, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); +stmt_ty _PyAST_Import(asdl_alias_seq * names, int is_lazy, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); stmt_ty _PyAST_ImportFrom(identifier module, asdl_alias_seq * names, int level, - int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); + int is_lazy, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); stmt_ty _PyAST_Global(asdl_identifier_seq * names, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); stmt_ty _PyAST_Nonlocal(asdl_identifier_seq * names, int lineno, int diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index d4ac419f51d..32c12fb5875 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -161,6 +161,7 @@ struct ast_state { PyObject *__module__; PyObject *_attributes; PyObject *_fields; + PyObject *abstract_types; PyObject *alias_type; PyObject *annotation; PyObject *arg; @@ -205,6 +206,7 @@ struct ast_state { PyObject *id; PyObject *ifs; PyObject *is_async; + PyObject *is_lazy; PyObject *items; PyObject *iter; PyObject *key; diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 454c8dde031..38dd82f6fc8 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -12,6 +12,7 @@ extern "C" { #include #include #include "pycore_structs.h" // _Py_BackoffCounter +#include "pycore_interp_structs.h" // _PyOptimizationConfig /* 16-bit countdown counters using exponential backoff. @@ -22,33 +23,48 @@ extern "C" { Another use is for the Tier 2 optimizer to decide when to create a new Tier 2 trace (executor). Again, exponential backoff is used. - The 16-bit counter is structured as a 12-bit unsigned 'value' - and a 4-bit 'backoff' field. When resetting the counter, the + The 16-bit counter is structured as a 13-bit unsigned 'value' + and a 3-bit 'backoff' field. When resetting the counter, the backoff field is incremented (until it reaches a limit) and the - value is set to a bit mask representing the value 2**backoff - 1. - The maximum backoff is 12 (the number of bits in the value). + value is set to a bit mask representing some prime value - 1. + New values and backoffs for each backoff are calculated once + at compile time and saved to value_and_backoff_next table. + The maximum backoff is 6, since 7 is an UNREACHABLE_BACKOFF. There is an exceptional value which must not be updated, 0xFFFF. */ -#define BACKOFF_BITS 4 -#define MAX_BACKOFF 12 -#define UNREACHABLE_BACKOFF 15 +#define BACKOFF_BITS 3 +#define BACKOFF_MASK 7 +#define MAX_BACKOFF 6 +#define UNREACHABLE_BACKOFF 7 +#define MAX_VALUE 0x1FFF -static inline bool -is_unreachable_backoff_counter(_Py_BackoffCounter counter) -{ - return counter.value_and_backoff == UNREACHABLE_BACKOFF; -} +#define MAKE_VALUE_AND_BACKOFF(value, backoff) \ + ((uint16_t)(((value) << BACKOFF_BITS) | (backoff))) + +// For previous backoff b we use value x such that +// x + 1 is near to 2**(2*b+1) and x + 1 is prime. +static const uint16_t value_and_backoff_next[] = { + MAKE_VALUE_AND_BACKOFF(1, 1), + MAKE_VALUE_AND_BACKOFF(6, 2), + MAKE_VALUE_AND_BACKOFF(30, 3), + MAKE_VALUE_AND_BACKOFF(126, 4), + MAKE_VALUE_AND_BACKOFF(508, 5), + MAKE_VALUE_AND_BACKOFF(2052, 6), + // We use the same backoff counter for all backoffs >= MAX_BACKOFF. + MAKE_VALUE_AND_BACKOFF(8190, 6), + MAKE_VALUE_AND_BACKOFF(8190, 6), +}; static inline _Py_BackoffCounter make_backoff_counter(uint16_t value, uint16_t backoff) { - assert(backoff <= 15); - assert(value <= 0xFFF); - _Py_BackoffCounter result; - result.value_and_backoff = (value << BACKOFF_BITS) | backoff; - return result; + assert(backoff <= UNREACHABLE_BACKOFF); + assert(value <= MAX_VALUE); + return ((_Py_BackoffCounter){ + .value_and_backoff = MAKE_VALUE_AND_BACKOFF(value, backoff) + }); } static inline _Py_BackoffCounter @@ -62,14 +78,11 @@ forge_backoff_counter(uint16_t counter) static inline _Py_BackoffCounter restart_backoff_counter(_Py_BackoffCounter counter) { - assert(!is_unreachable_backoff_counter(counter)); - int backoff = counter.value_and_backoff & 15; - if (backoff < MAX_BACKOFF) { - return make_backoff_counter((1 << (backoff + 1)) - 1, backoff + 1); - } - else { - return make_backoff_counter((1 << MAX_BACKOFF) - 1, MAX_BACKOFF); - } + uint16_t backoff = counter.value_and_backoff & BACKOFF_MASK; + assert(backoff <= MAX_BACKOFF); + return ((_Py_BackoffCounter){ + .value_and_backoff = value_and_backoff_next[backoff] + }); } static inline _Py_BackoffCounter @@ -95,31 +108,60 @@ backoff_counter_triggers(_Py_BackoffCounter counter) return counter.value_and_backoff < UNREACHABLE_BACKOFF; } +static inline _Py_BackoffCounter +trigger_backoff_counter(void) +{ + _Py_BackoffCounter result; + result.value_and_backoff = 0; + return result; +} + // Initial JUMP_BACKWARD counter. // Must be larger than ADAPTIVE_COOLDOWN_VALUE, otherwise when JIT code is // invalidated we may construct a new trace before the bytecode has properly // re-specialized: -#define JUMP_BACKWARD_INITIAL_VALUE 4095 -#define JUMP_BACKWARD_INITIAL_BACKOFF 12 +// Note: this should be a prime number-1. This increases the likelihood of +// finding a "good" loop iteration to trace. +// For example, 4095 does not work for the nqueens benchmark on pyperformance +// as we always end up tracing the loop iteration's +// exhaustion iteration. Which aborts our current tracer. +#define JUMP_BACKWARD_INITIAL_VALUE 4000 +#define JUMP_BACKWARD_INITIAL_BACKOFF 6 static inline _Py_BackoffCounter -initial_jump_backoff_counter(void) +initial_jump_backoff_counter(_PyOptimizationConfig *opt_config) { - return make_backoff_counter(JUMP_BACKWARD_INITIAL_VALUE, - JUMP_BACKWARD_INITIAL_BACKOFF); + return make_backoff_counter( + opt_config->jump_backward_initial_value, + opt_config->jump_backward_initial_backoff); +} + +// This needs to be around 2-4x of JUMP_BACKWARD_INITIAL_VALUE +// The reasoning is that we always want loop traces to form and inline +// functions before functions themselves warm up and link to them instead +// of inlining. +#define RESUME_INITIAL_VALUE 8190 +#define RESUME_INITIAL_BACKOFF 6 +static inline _Py_BackoffCounter +initial_resume_backoff_counter(_PyOptimizationConfig *opt_config) +{ + return make_backoff_counter( + opt_config->resume_initial_value, + opt_config->resume_initial_backoff); } /* Initial exit temperature. * Must be larger than ADAPTIVE_COOLDOWN_VALUE, * otherwise when a side exit warms up we may construct * a new trace before the Tier 1 code has properly re-specialized. */ -#define SIDE_EXIT_INITIAL_VALUE 4095 -#define SIDE_EXIT_INITIAL_BACKOFF 12 +#define SIDE_EXIT_INITIAL_VALUE 4000 +#define SIDE_EXIT_INITIAL_BACKOFF 6 static inline _Py_BackoffCounter -initial_temperature_backoff_counter(void) +initial_temperature_backoff_counter(_PyOptimizationConfig *opt_config) { - return make_backoff_counter(SIDE_EXIT_INITIAL_VALUE, - SIDE_EXIT_INITIAL_BACKOFF); + return make_backoff_counter( + opt_config->side_exit_initial_value, + opt_config->side_exit_initial_backoff); } /* Unreachable backoff counter. */ diff --git a/Include/internal/pycore_blocks_output_buffer.h b/Include/internal/pycore_blocks_output_buffer.h index 573e10359b7..322c1e93344 100644 --- a/Include/internal/pycore_blocks_output_buffer.h +++ b/Include/internal/pycore_blocks_output_buffer.h @@ -45,12 +45,14 @@ extern "C" { #endif typedef struct { - // List of bytes objects - PyObject *list; + // Bytes writer managing output buffer + PyBytesWriter *writer; // Number of whole allocated size Py_ssize_t allocated; - // Max length of the buffer, negative number means unlimited length. + // Max length of the buffer, negative number means unlimited length Py_ssize_t max_length; + // Number of blocks of bytes. Used to calculate next allocation size + size_t num_blocks; } _BlocksOutputBuffer; static const char unable_allocate_msg[] = "Unable to allocate output buffer."; @@ -107,11 +109,10 @@ _BlocksOutputBuffer_InitAndGrow(_BlocksOutputBuffer *buffer, const Py_ssize_t max_length, void **next_out) { - PyObject *b; Py_ssize_t block_size; - // ensure .list was set to NULL - assert(buffer->list == NULL); + // ensure .writer was set to NULL + assert(buffer->writer == NULL); // get block size if (0 <= max_length && max_length < BUFFER_BLOCK_SIZE[0]) { @@ -120,25 +121,17 @@ _BlocksOutputBuffer_InitAndGrow(_BlocksOutputBuffer *buffer, block_size = BUFFER_BLOCK_SIZE[0]; } - // the first block - b = PyBytes_FromStringAndSize(NULL, block_size); - if (b == NULL) { + buffer->writer = PyBytesWriter_Create(block_size); + if (buffer->writer == NULL) { return -1; } - // create the list - buffer->list = PyList_New(1); - if (buffer->list == NULL) { - Py_DECREF(b); - return -1; - } - PyList_SET_ITEM(buffer->list, 0, b); - // set variables buffer->allocated = block_size; buffer->max_length = max_length; + buffer->num_blocks = 1; - *next_out = PyBytes_AS_STRING(b); + *next_out = PyBytesWriter_GetData(buffer->writer); return block_size; } @@ -155,31 +148,21 @@ _BlocksOutputBuffer_InitWithSize(_BlocksOutputBuffer *buffer, const Py_ssize_t init_size, void **next_out) { - PyObject *b; - // ensure .list was set to NULL - assert(buffer->list == NULL); + // ensure .writer was set to NULL + assert(buffer->writer == NULL); - // the first block - b = PyBytes_FromStringAndSize(NULL, init_size); - if (b == NULL) { - PyErr_SetString(PyExc_MemoryError, unable_allocate_msg); + buffer->writer = PyBytesWriter_Create(init_size); + if (buffer->writer == NULL) { return -1; } - // create the list - buffer->list = PyList_New(1); - if (buffer->list == NULL) { - Py_DECREF(b); - return -1; - } - PyList_SET_ITEM(buffer->list, 0, b); - // set variables buffer->allocated = init_size; buffer->max_length = -1; + buffer->num_blocks = 1; - *next_out = PyBytes_AS_STRING(b); + *next_out = PyBytesWriter_GetData(buffer->writer); return init_size; } @@ -193,8 +176,6 @@ _BlocksOutputBuffer_Grow(_BlocksOutputBuffer *buffer, void **next_out, const Py_ssize_t avail_out) { - PyObject *b; - const Py_ssize_t list_len = Py_SIZE(buffer->list); Py_ssize_t block_size; // ensure no gaps in the data @@ -205,11 +186,10 @@ _BlocksOutputBuffer_Grow(_BlocksOutputBuffer *buffer, } // get block size - if (list_len < (Py_ssize_t) Py_ARRAY_LENGTH(BUFFER_BLOCK_SIZE)) { - block_size = BUFFER_BLOCK_SIZE[list_len]; - } else { - block_size = BUFFER_BLOCK_SIZE[Py_ARRAY_LENGTH(BUFFER_BLOCK_SIZE) - 1]; - } + size_t maxblock = Py_ARRAY_LENGTH(BUFFER_BLOCK_SIZE); + assert(maxblock >= 1); + size_t block_index = Py_MIN(buffer->num_blocks, maxblock - 1); + block_size = BUFFER_BLOCK_SIZE[block_index]; // check max_length if (buffer->max_length >= 0) { @@ -229,22 +209,19 @@ _BlocksOutputBuffer_Grow(_BlocksOutputBuffer *buffer, return -1; } - // create the block - b = PyBytes_FromStringAndSize(NULL, block_size); - if (b == NULL) { + if (PyBytesWriter_Grow(buffer->writer, block_size)) { PyErr_SetString(PyExc_MemoryError, unable_allocate_msg); return -1; } - if (PyList_Append(buffer->list, b) < 0) { - Py_DECREF(b); - return -1; - } - Py_DECREF(b); + + Py_ssize_t current_size = buffer->allocated; // set variables buffer->allocated += block_size; + buffer->num_blocks += 1; - *next_out = PyBytes_AS_STRING(b); + char *data = PyBytesWriter_GetData(buffer->writer); + *next_out = data + current_size; return block_size; } @@ -265,54 +242,20 @@ static inline PyObject * _BlocksOutputBuffer_Finish(_BlocksOutputBuffer *buffer, const Py_ssize_t avail_out) { - PyObject *result, *block; - const Py_ssize_t list_len = Py_SIZE(buffer->list); - - // fast path for single block - if ((list_len == 1 && avail_out == 0) || - (list_len == 2 && Py_SIZE(PyList_GET_ITEM(buffer->list, 1)) == avail_out)) - { - block = PyList_GET_ITEM(buffer->list, 0); - Py_INCREF(block); - - Py_CLEAR(buffer->list); - return block; - } - - // final bytes object - result = PyBytes_FromStringAndSize(NULL, buffer->allocated - avail_out); - if (result == NULL) { - PyErr_SetString(PyExc_MemoryError, unable_allocate_msg); - return NULL; - } - - // memory copy - if (list_len > 0) { - char *posi = PyBytes_AS_STRING(result); - - // blocks except the last one - Py_ssize_t i = 0; - for (; i < list_len-1; i++) { - block = PyList_GET_ITEM(buffer->list, i); - memcpy(posi, PyBytes_AS_STRING(block), Py_SIZE(block)); - posi += Py_SIZE(block); - } - // the last block - block = PyList_GET_ITEM(buffer->list, i); - memcpy(posi, PyBytes_AS_STRING(block), Py_SIZE(block) - avail_out); - } else { - assert(Py_SIZE(result) == 0); - } - - Py_CLEAR(buffer->list); - return result; + PyObject *obj; + assert(buffer->writer != NULL); + obj = PyBytesWriter_FinishWithSize(buffer->writer, + buffer->allocated - avail_out); + buffer->writer = NULL; + return obj; } /* Clean up the buffer when an error occurred. */ static inline void _BlocksOutputBuffer_OnError(_BlocksOutputBuffer *buffer) { - Py_CLEAR(buffer->list); + PyBytesWriter_Discard(buffer->writer); + buffer->writer = NULL; } #ifdef __cplusplus diff --git a/Include/internal/pycore_bytes_methods.h b/Include/internal/pycore_bytes_methods.h index 059dc2599bb..3e1474c1c01 100644 --- a/Include/internal/pycore_bytes_methods.h +++ b/Include/internal/pycore_bytes_methods.h @@ -47,6 +47,9 @@ extern PyObject *_Py_bytes_endswith(const char *str, Py_ssize_t len, /* The maketrans() static method. */ extern PyObject* _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to); +/* Helper for repr(bytes) and repr(bytearray). */ +extern PyObject *_Py_bytes_repr(const char *, Py_ssize_t, int, const char *); + /* Shared __doc__ strings. */ extern const char _Py_isspace__doc__[]; extern const char _Py_isalpha__doc__[]; diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index 8ea9b3ebb88..177e6d10134 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -14,6 +14,11 @@ extern PyObject* _PyBytes_FormatEx( PyObject *args, int use_bytearray); +/* Concatenate two bytes objects. Used as the sq_concat slot and by the + * specializing interpreter. Unlike PyBytes_Concat(), this returns a new + * reference rather than modifying its first argument in place. */ +extern PyObject* _PyBytes_Concat(PyObject *a, PyObject *b); + extern PyObject* _PyBytes_FromHex( PyObject *string, int use_bytearray); @@ -60,88 +65,46 @@ PyAPI_FUNC(void) _PyBytes_Repeat(char* dest, Py_ssize_t len_dest, const char* src, Py_ssize_t len_src); -/* --- _PyBytesWriter ----------------------------------------------------- */ +/* _PyBytesObject_SIZE gives the basic size of a bytes object; any memory allocation + for a bytes object of length n should request PyBytesObject_SIZE + n bytes. -/* The _PyBytesWriter structure is big: it contains an embedded "stack buffer". - A _PyBytesWriter variable must be declared at the end of variables in a - function to optimize the memory allocation on the stack. */ -typedef struct { - /* bytes, bytearray or NULL (when the small buffer is used) */ - PyObject *buffer; + Using _PyBytesObject_SIZE instead of sizeof(PyBytesObject) saves + 3 or 7 bytes per bytes object allocation on a typical system. +*/ +#define _PyBytesObject_SIZE (offsetof(PyBytesObject, ob_sval) + 1) - /* Number of allocated size. */ - Py_ssize_t allocated; +/* --- PyBytesWriter ------------------------------------------------------ */ - /* Minimum number of allocated bytes, - incremented by _PyBytesWriter_Prepare() */ - Py_ssize_t min_size; - - /* If non-zero, use a bytearray instead of a bytes object for buffer. */ +struct PyBytesWriter { + char small_buffer[256]; + PyObject *obj; + Py_ssize_t size; int use_bytearray; - - /* If non-zero, overallocate the buffer (default: 0). - This flag must be zero if use_bytearray is non-zero. */ int overallocate; +}; - /* Stack buffer */ - int use_small_buffer; - char small_buffer[512]; -} _PyBytesWriter; +// Export for '_testcapi' shared extension +PyAPI_FUNC(PyBytesWriter*) _PyBytesWriter_CreateByteArray(Py_ssize_t size); -/* Initialize a bytes writer +static inline Py_ssize_t +_PyBytesWriter_GetSize(PyBytesWriter *writer) +{ + return writer->size; +} - By default, the overallocation is disabled. Set the overallocate attribute - to control the allocation of the buffer. - - Export _PyBytesWriter API for '_pickle' shared extension. */ -PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer); - -/* Get the buffer content and reset the writer. - Return a bytes object, or a bytearray object if use_bytearray is non-zero. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer, - void *str); - -/* Deallocate memory of a writer (clear its internal buffer). */ -PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer); - -/* Allocate the buffer to write size bytes. - Return the pointer to the beginning of buffer data. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_Alloc(_PyBytesWriter *writer, - Py_ssize_t size); - -/* Ensure that the buffer is large enough to write *size* bytes. - Add size to the writer minimum size (min_size attribute). - - str is the current pointer inside the buffer. - Return the updated current pointer inside the buffer. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_Prepare(_PyBytesWriter *writer, - void *str, - Py_ssize_t size); - -/* Resize the buffer to make it larger. - The new buffer may be larger than size bytes because of overallocation. - Return the updated current pointer inside the buffer. - Raise an exception and return NULL on error. - - Note: size must be greater than the number of allocated bytes in the writer. - - This function doesn't use the writer minimum size (min_size attribute). - - See also _PyBytesWriter_Prepare(). - */ -PyAPI_FUNC(void*) _PyBytesWriter_Resize(_PyBytesWriter *writer, - void *str, - Py_ssize_t size); - -/* Write bytes. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer, - void *str, - const void *bytes, - Py_ssize_t size); +static inline char* +_PyBytesWriter_GetData(PyBytesWriter *writer) +{ + if (writer->obj == NULL) { + return writer->small_buffer; + } + else if (writer->use_bytearray) { + return PyByteArray_AS_STRING(writer->obj); + } + else { + return PyBytes_AS_STRING(writer->obj); + } +} #ifdef __cplusplus } diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index 32ac3d17f22..a9db8860e91 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -64,39 +64,14 @@ PyAPI_FUNC(PyObject*) _PyObject_CallMethod( PyObject *name, const char *format, ...); -extern PyObject* _PyObject_CallMethodIdObjArgs( - PyObject *obj, - _Py_Identifier *name, - ...); - -static inline PyObject * -_PyObject_VectorcallMethodId( - _Py_Identifier *name, PyObject *const *args, - size_t nargsf, PyObject *kwnames) -{ - PyObject *oname = _PyUnicode_FromId(name); /* borrowed */ - if (!oname) { - return _Py_NULL; - } - return PyObject_VectorcallMethod(oname, args, nargsf, kwnames); -} - -static inline PyObject * -_PyObject_CallMethodIdNoArgs(PyObject *self, _Py_Identifier *name) -{ - size_t nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; - return _PyObject_VectorcallMethodId(name, &self, nargsf, _Py_NULL); -} - -static inline PyObject * -_PyObject_CallMethodIdOneArg(PyObject *self, _Py_Identifier *name, PyObject *arg) -{ - PyObject *args[2] = {self, arg}; - size_t nargsf = 2 | PY_VECTORCALL_ARGUMENTS_OFFSET; - assert(arg != NULL); - return _PyObject_VectorcallMethodId(name, args, nargsf, _Py_NULL); -} +extern PyObject *_PyObject_VectorcallPrepend( + PyThreadState *tstate, + PyObject *callable, + PyObject *arg, + PyObject *const *args, + size_t nargsf, + PyObject *kwnames); /* === Vectorcall protocol (PEP 590) ============================= */ @@ -186,17 +161,18 @@ _PyObject_CallNoArgs(PyObject *func) { } -extern PyObject *const * +PyAPI_FUNC(PyObject *const *) _PyStack_UnpackDict(PyThreadState *tstate, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs, PyObject **p_kwnames); -extern void _PyStack_UnpackDict_Free( +// Exported for external JIT support +PyAPI_FUNC(void) _PyStack_UnpackDict_Free( PyObject *const *stack, Py_ssize_t nargs, PyObject *kwnames); -extern void _PyStack_UnpackDict_FreeNoDecRef( +PyAPI_FUNC(void) _PyStack_UnpackDict_FreeNoDecRef( PyObject *const *stack, PyObject *kwnames); diff --git a/Include/internal/pycore_cell.h b/Include/internal/pycore_cell.h index cef01e80514..d0d45a23436 100644 --- a/Include/internal/pycore_cell.h +++ b/Include/internal/pycore_cell.h @@ -53,7 +53,7 @@ _PyCell_GetStackRef(PyCellObject *cell) { PyObject *value; #ifdef Py_GIL_DISABLED - value = _Py_atomic_load_ptr(&cell->ob_ref); + value = _PyObject_CAST(_Py_atomic_load_ptr(&cell->ob_ref)); if (value == NULL) { return PyStackRef_NULL; } diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a03fe42668f..fd4221f0816 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -33,8 +33,6 @@ extern int _PyEval_SetOpcodeTrace(PyFrameObject *f, bool enable); // Export for 'array' shared extension PyAPI_FUNC(PyObject*) _PyEval_GetBuiltin(PyObject *); -extern PyObject* _PyEval_GetBuiltinId(_Py_Identifier *); - extern void _PyEval_SetSwitchInterval(unsigned long microseconds); extern unsigned long _PyEval_GetSwitchInterval(void); @@ -96,7 +94,7 @@ typedef struct { void* (*init_state)(void); // Callback to register every trampoline being created void (*write_state)(void* state, const void *code_addr, - unsigned int code_size, PyCodeObject* code); + size_t code_size, PyCodeObject* code); // Callback to free the trampoline state int (*free_state)(void* state); } _PyPerf_Callbacks; @@ -105,12 +103,15 @@ extern int _PyPerfTrampoline_SetCallbacks(_PyPerf_Callbacks *); extern void _PyPerfTrampoline_GetCallbacks(_PyPerf_Callbacks *); extern int _PyPerfTrampoline_Init(int activate); extern int _PyPerfTrampoline_Fini(void); -extern void _PyPerfTrampoline_FreeArenas(void); extern int _PyIsPerfTrampolineActive(void); extern PyStatus _PyPerfTrampoline_AfterFork_Child(void); #ifdef PY_HAVE_PERF_TRAMPOLINE extern _PyPerf_Callbacks _Py_perfmap_callbacks; extern _PyPerf_Callbacks _Py_perfmap_jit_callbacks; +extern void _PyPerfJit_WriteNamedCode(const void *code_addr, + size_t code_size, + const char *entry, + const char *filename); #endif static inline PyObject* @@ -124,18 +125,11 @@ _PyEval_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwfl } #ifdef _Py_TIER2 -#ifdef _Py_JIT -_Py_CODEUNIT *_Py_LazyJitTrampoline( - struct _PyExecutorObject *current_executor, _PyInterpreterFrame *frame, - _PyStackRef *stack_pointer, PyThreadState *tstate -); -#else _Py_CODEUNIT *_PyTier2Interpreter( struct _PyExecutorObject *current_executor, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate ); #endif -#endif extern _PyJitEntryFuncPtr _Py_jit_entry; @@ -217,7 +211,14 @@ extern void _PyEval_DeactivateOpCache(void); static inline int _Py_MakeRecCheck(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - return here_addr < _tstate->c_stack_soft_limit; + // Overflow if stack pointer is between soft limit and the base of the hardware stack. + // If it is below the hardware stack base, assume that we have the wrong stack limits, and do nothing. + // We could have the wrong stack limits because of limited platform support, or user-space threads. +#if _Py_STACK_GROWS_DOWN + return here_addr < _tstate->c_stack_soft_limit && here_addr >= _tstate->c_stack_soft_limit - 2 * _PyOS_STACK_MARGIN_BYTES; +#else + return here_addr > _tstate->c_stack_soft_limit && here_addr <= _tstate->c_stack_soft_limit + 2 * _PyOS_STACK_MARGIN_BYTES; +#endif } // Export for '_json' shared extension, used via _Py_EnterRecursiveCall() @@ -226,7 +227,7 @@ PyAPI_FUNC(int) _Py_CheckRecursiveCall( PyThreadState *tstate, const char *where); -int _Py_CheckRecursiveCallPy( +PyAPI_FUNC(int) _Py_CheckRecursiveCallPy( PyThreadState *tstate); static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate, @@ -245,12 +246,12 @@ static inline void _Py_LeaveRecursiveCallTstate(PyThreadState *tstate) { PyAPI_FUNC(void) _Py_InitializeRecursionLimits(PyThreadState *tstate); -static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate) { - uintptr_t here_addr = _Py_get_machine_stack_pointer(); - _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - assert(_tstate->c_stack_hard_limit != 0); - return here_addr <= _tstate->c_stack_soft_limit; -} +PyAPI_FUNC(int) _Py_ReachedRecursionLimit(PyThreadState *tstate); + +// Export for test_peg_generator +PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin( + PyThreadState *tstate, + int margin_count); static inline void _Py_LeaveRecursiveCall(void) { } @@ -273,6 +274,9 @@ PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); and asynchronous exception */ PyAPI_FUNC(int) _Py_HandlePending(PyThreadState *tstate); +/* Raise exception set by PyThreadState_SetAsyncExc, if any */ +PyAPI_FUNC(int) _PyEval_RaiseAsyncExc(PyThreadState *tstate); + extern PyObject * _PyEval_GetFrameLocals(void); typedef PyObject *(*conversion_func)(PyObject *); @@ -295,12 +299,25 @@ PyAPI_FUNC(int) _PyEval_ExceptionGroupMatch(_PyInterpreterFrame *, PyObject* exc PyAPI_FUNC(void) _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg); PyAPI_FUNC(void) _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj); PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg); -PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs); +PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs, PyObject *dupkey); PyAPI_FUNC(PyObject *) _PyEval_ImportFrom(PyThreadState *, PyObject *, PyObject *); -PyAPI_FUNC(PyObject *) _PyEval_ImportName(PyThreadState *, _PyInterpreterFrame *, PyObject *, PyObject *, PyObject *); + +PyAPI_FUNC(PyObject *) _PyEval_LazyImportName( + PyThreadState *tstate, PyObject *builtins, PyObject *globals, + PyObject *locals, PyObject *name, PyObject *fromlist, PyObject *level, + int lazy); +PyAPI_FUNC(PyObject *) _PyEval_LazyImportFrom( + PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *v, PyObject *name); +PyAPI_FUNC(PyObject *) _PyEval_ImportName( + PyThreadState *tstate, PyObject *builtins, PyObject *globals, + PyObject *locals, PyObject *name, PyObject *fromlist, PyObject *level); +PyObject * _PyEval_ImportNameWithImport( + PyThreadState *tstate, PyObject *import_func, PyObject *globals, + PyObject *locals, PyObject *name, PyObject *fromlist, PyObject *level); PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); +PyAPI_FUNC(bool) _PyEval_NoToolsForUnwind(PyThreadState *tstate, _PyInterpreterFrame *frame); PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, _PyStackRef *sp); PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch); @@ -313,6 +330,7 @@ PyAPI_FUNC(PyObject *) _PyEval_GetAwaitable(PyObject *iterable, int oparg); PyAPI_FUNC(PyObject *) _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *name); PyAPI_FUNC(int) _Py_Check_ArgsIterable(PyThreadState *tstate, PyObject *func, PyObject *args); +PyAPI_FUNC(_PyStackRef) _PyEval_GetIter(_PyStackRef iterable, _PyStackRef *null_or_index, int yield_from); /* * Indicate whether a special method of given 'oparg' can use the (improved) @@ -362,8 +380,6 @@ _Py_eval_breaker_bit_is_set(PyThreadState *tstate, uintptr_t bit) void _Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit); void _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit); -PyAPI_FUNC(_PyStackRef) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef right, double value); - #ifndef Py_SUPPORTS_REMOTE_DEBUG #if defined(__APPLE__) #include @@ -391,6 +407,87 @@ _PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame #define SPECIAL___AEXIT__ 3 #define SPECIAL_MAX 3 +PyAPI_DATA(const _Py_CODEUNIT *) _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR; + +/* Helper functions for large uops */ + +PyAPI_FUNC(PyObject *) +_Py_VectorCall_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args, + _PyStackRef kwnames); + +PyAPI_FUNC(PyObject*) +_Py_VectorCallInstrumentation_StackRefSteal( + _PyStackRef callable, + _PyStackRef* arguments, + int total_args, + _PyStackRef kwnames, + bool call_instrumentation, + _PyInterpreterFrame* frame, + _Py_CODEUNIT* this_instr, + PyThreadState* tstate); + +PyAPI_FUNC(PyObject *) +_Py_BuiltinCallFast_StackRef( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_BuiltinCallFastWithKeywords_StackRef( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_PyCallMethodDescriptorFast_StackRef( + _PyStackRef callable, + PyCFunctionFast cfunc, + PyObject *self, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_PyCallMethodDescriptorFastWithKeywords_StackRef( + _PyStackRef callable, + PyCFunctionFastWithKeywords cfunc, + PyObject *self, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_CallBuiltinClass_StackRef( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_BuildString_StackRefSteal( + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_BuildMap_StackRefSteal( + _PyStackRef *arguments, + int half_args); + +PyAPI_FUNC(void) +_Py_assert_within_stack_bounds( + _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + const char *filename, int lineno); + +PyAPI_FUNC(_PyStackRef) +_Py_LoadAttr_StackRefSteal( + PyThreadState *tstate, _PyStackRef owner, + PyObject *name, _PyStackRef *self_or_null); + +// Like PyMapping_GetOptionalItem, but returns the PyObject* instead of taking +// it as an out parameter. This helps MSVC's escape analysis when used with +// tail calling. +PyAPI_FUNC(PyObject*) _PyMapping_GetOptionalItem2(PyObject* obj, PyObject* key, int* err); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 8e1415f27b6..4584f1bde79 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -141,6 +141,12 @@ typedef struct { #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache) +typedef struct { + _Py_BackoffCounter counter; +} _PyGetIterCache; + +#define INLINE_CACHE_ENTRIES_GET_ITER CACHE_ENTRIES(_PyGetIterCache) + typedef struct { _Py_BackoffCounter counter; } _PySendCache; @@ -160,6 +166,12 @@ typedef struct { #define INLINE_CACHE_ENTRIES_CONTAINS_OP CACHE_ENTRIES(_PyContainsOpCache) +typedef struct { + _Py_BackoffCounter counter; +} _PyCallFunctionExCache; + +#define INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX CACHE_ENTRIES(_PyCallFunctionExCache) + /* "Locals plus" for a code object is the set of locals + cell vars + * free vars. This relates to variable names as well as offsets into * the "fast locals" storage array of execution frames. The compiler @@ -262,7 +274,7 @@ extern PyObject* _PyCode_GetFreevars(PyCodeObject *); extern PyObject* _PyCode_GetCode(PyCodeObject *); /** API for initializing the line number tables. */ -extern int _PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds); +PyAPI_FUNC(int) _PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds); /** Out of process API for initializing the location table. */ extern void _PyLineTable_InitAddressRange( @@ -272,52 +284,53 @@ extern void _PyLineTable_InitAddressRange( PyCodeAddressRange *range); /** API for traversing the line number table. */ -extern int _PyLineTable_NextAddressRange(PyCodeAddressRange *range); +PyAPI_FUNC(int) _PyLineTable_NextAddressRange(PyCodeAddressRange *range); extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); +// Similar to PyCode_Addr2Line(), but return -1 if the code object is invalid +// and can be called without an attached tstate. Used by dump_frame() in +// Python/traceback.c. The function uses heuristics to detect freed memory, +// it's not 100% reliable. +extern int _PyCode_SafeAddr2Line(PyCodeObject *co, int addr); + + /** API for executors */ extern void _PyCode_Clear_Executors(PyCodeObject *code); -#ifdef Py_GIL_DISABLED -// gh-115999 tracks progress on addressing this. -#define ENABLE_SPECIALIZATION 0 -// Use this to enable specialization families once they are thread-safe. All -// uses will be replaced with ENABLE_SPECIALIZATION once all families are -// thread-safe. -#define ENABLE_SPECIALIZATION_FT 1 -#else #define ENABLE_SPECIALIZATION 1 -#define ENABLE_SPECIALIZATION_FT ENABLE_SPECIALIZATION -#endif -/* Specialization functions */ +/* Specialization functions, these are exported only for other re-generated + * interpreters to call */ -extern void _Py_Specialize_LoadSuperAttr(_PyStackRef global_super, _PyStackRef cls, +PyAPI_FUNC(void) _Py_Specialize_LoadSuperAttr(_PyStackRef global_super, _PyStackRef cls, _Py_CODEUNIT *instr, int load_method); -extern void _Py_Specialize_LoadAttr(_PyStackRef owner, _Py_CODEUNIT *instr, +PyAPI_FUNC(void) _Py_Specialize_LoadAttr(_PyStackRef owner, _Py_CODEUNIT *instr, PyObject *name); -extern void _Py_Specialize_StoreAttr(_PyStackRef owner, _Py_CODEUNIT *instr, +PyAPI_FUNC(void) _Py_Specialize_StoreAttr(_PyStackRef owner, _Py_CODEUNIT *instr, PyObject *name); -extern void _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, +PyAPI_FUNC(void) _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name); -extern void _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, +PyAPI_FUNC(void) _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, _Py_CODEUNIT *instr); -extern void _Py_Specialize_Call(_PyStackRef callable, _Py_CODEUNIT *instr, - int nargs); -extern void _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr, +PyAPI_FUNC(void) _Py_Specialize_Call(_PyStackRef callable, _PyStackRef self_or_null, + _Py_CODEUNIT *instr, int nargs); +PyAPI_FUNC(void) _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr, int nargs); -extern void _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, +PyAPI_FUNC(void) _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg, _PyStackRef *locals); -extern void _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs, +PyAPI_FUNC(void) _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_UnpackSequence(_PyStackRef seq, _Py_CODEUNIT *instr, +PyAPI_FUNC(void) _Py_Specialize_UnpackSequence(_PyStackRef seq, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr); -extern void _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr); -extern void _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr); -extern void _Py_GatherStats_GetIter(_PyStackRef iterable); +PyAPI_FUNC(void) _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT *instr, int oparg); +PyAPI_FUNC(void) _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr); +PyAPI_FUNC(void) _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr); +PyAPI_FUNC(void) _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr); +PyAPI_FUNC(void) _Py_GatherStats_GetIter(_PyStackRef iterable); +PyAPI_FUNC(void) _Py_Specialize_CallFunctionEx(_PyStackRef func_st, _Py_CODEUNIT *instr); +PyAPI_FUNC(void) _Py_Specialize_Resume(_Py_CODEUNIT *instr, PyThreadState *tstate, _PyInterpreterFrame *frame); +PyAPI_FUNC(void) _Py_Specialize_GetIter(_PyStackRef iterable, _Py_CODEUNIT *instr); // Utility functions for reading/writing 32/64-bit values in the inline caches. // Great care should be taken to ensure that these functions remain correct and @@ -490,6 +503,18 @@ typedef struct { int oparg; binaryopguardfunc guard; binaryopactionfunc action; + /* Static type of the result, or NULL if unknown. Used by the tier 2 + optimizer to propagate type information through _BINARY_OP_EXTEND. */ + PyTypeObject *result_type; + /* Nonzero iff `action` always returns a freshly allocated object (not + aliased to either operand). Used by the tier 2 optimizer to enable + inplace follow-up ops. */ + int result_unique; + /* Expected types of the left and right operands. Used by the tier 2 + optimizer to eliminate _GUARD_BINARY_OP_EXTEND when the operand + types are already known. NULL means unknown/don't eliminate. */ + PyTypeObject *lhs_type; + PyTypeObject *rhs_type; } _PyBinaryOpSpecializationDescr; /* Comparison bit masks. */ @@ -512,7 +537,7 @@ typedef struct { #define COMPARISON_NOT_EQUALS (COMPARISON_UNORDERED | COMPARISON_LESS_THAN | COMPARISON_GREATER_THAN) -extern int _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); +PyAPI_FUNC(int) _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); extern _Py_CODEUNIT _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); @@ -548,7 +573,7 @@ _PyCode_GetTLBCFast(PyThreadState *tstate, PyCodeObject *co) // Return a pointer to the thread-local bytecode for the current thread, // creating it if necessary. -extern _Py_CODEUNIT *_PyCode_GetTLBC(PyCodeObject *co); +PyAPI_FUNC(_Py_CODEUNIT *) _PyCode_GetTLBC(PyCodeObject *co); // Reserve an index for the current thread into thread-local bytecode // arrays @@ -660,6 +685,15 @@ PyAPI_FUNC(int) _PyCode_VerifyStateless( PyAPI_FUNC(int) _PyCode_CheckPureFunction(PyCodeObject *, const char **); PyAPI_FUNC(int) _PyCode_ReturnsOnlyNone(PyCodeObject *); +/* Create a comparable key used to compare constants taking in account the + * object type. It is used to make sure types are not coerced (e.g., float and + * complex) _and_ to distinguish 0.0 from -0.0 e.g. on IEEE platforms + * + * Return (type(obj), obj, ...): a tuple with variable size (at least 2 items) + * depending on the type and the value. The type is the first item to not + * compare bytes and str which can raise a BytesWarning exception. */ +extern PyObject* _PyCode_ConstantKey(PyObject *obj); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index c18e04bf67a..911cc1f10f1 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -32,7 +32,8 @@ PyAPI_FUNC(PyCodeObject*) _PyAST_Compile( PyObject *filename, PyCompilerFlags *flags, int optimize, - struct _arena *arena); + struct _arena *arena, + PyObject *module); /* AST preprocessing */ extern int _PyCompile_AstPreprocess( @@ -41,7 +42,8 @@ extern int _PyCompile_AstPreprocess( PyCompilerFlags *flags, int optimize, struct _arena *arena, - int syntax_check_only); + int syntax_check_only, + PyObject *module); extern int _PyAST_Preprocess( struct _mod *, @@ -49,7 +51,9 @@ extern int _PyAST_Preprocess( PyObject *filename, int optimize, int ff_features, - int syntax_check_only); + int syntax_check_only, + int enable_warnings, + PyObject *module); typedef struct { @@ -127,6 +131,7 @@ int _PyCompile_PushFBlock(struct _PyCompiler *c, _Py_SourceLocation loc, void _PyCompile_PopFBlock(struct _PyCompiler *c, enum _PyCompile_FBlockType t, _PyJumpTargetLabel block_label); _PyCompile_FBlockInfo *_PyCompile_TopFBlock(struct _PyCompiler *c); +bool _PyCompile_InExceptionHandler(struct _PyCompiler *c); int _PyCompile_EnterScope(struct _PyCompiler *c, identifier name, int scope_type, void *key, int lineno, PyObject *private, diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index c77ef7910c0..a833f790a62 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -55,5 +55,8 @@ struct _pycontexttokenobject { // Export for '_testcapi' shared extension PyAPI_FUNC(PyObject*) _PyContext_NewHamtForTests(void); +PyAPI_FUNC(int) _PyContext_Enter(PyThreadState *ts, PyObject *octx); +PyAPI_FUNC(int) _PyContext_Exit(PyThreadState *ts, PyObject *octx); + #endif /* !Py_INTERNAL_CONTEXT_H */ diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 2601de40737..2a2846b1296 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -32,7 +32,7 @@ extern "C" { const bool _should_lock_cs = PyList_CheckExact(_orig_seq); \ PyCriticalSection _cs; \ if (_should_lock_cs) { \ - _PyCriticalSection_Begin(&_cs, _orig_seq); \ + PyCriticalSection_Begin(&_cs, _orig_seq); \ } # define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() \ @@ -50,6 +50,15 @@ extern "C" { // Asserts that the mutex for the given object is locked. The mutex must // be held by the top-most critical section otherwise there's the // possibility that the mutex would be swalled out in some code paths. +// +// NOTE: We use Py_REFCNT(op) != 1 instead of +// !PyUnstable_Object_IsUniquelyReferenced(op) because the free threading +// GC can change an object's ob_tid (it overwrites ob_tid and later +// restores it from the mimalloc segment). This means +// PyUnstable_Object_IsUniquelyReferenced() may spuriously return false +// after a GC collection, even though the thread may still have exclusive +// access to the object. The refcount check is a looser but still catches +// most misuses. #ifdef Py_DEBUG # define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) \ @@ -77,10 +86,10 @@ _PyCriticalSection_Resume(PyThreadState *tstate); // (private) slow path for locking the mutex PyAPI_FUNC(void) -_PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m); +_PyCriticalSection_BeginSlow(PyThreadState *tstate, PyCriticalSection *c, PyMutex *m); PyAPI_FUNC(void) -_PyCriticalSection2_BeginSlow(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, +_PyCriticalSection2_BeginSlow(PyThreadState *tstate, PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, int is_m1_locked); PyAPI_FUNC(void) @@ -95,34 +104,30 @@ _PyCriticalSection_IsActive(uintptr_t tag) } static inline void -_PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m) +_PyCriticalSection_BeginMutex(PyThreadState *tstate, PyCriticalSection *c, PyMutex *m) { if (PyMutex_LockFast(m)) { - PyThreadState *tstate = _PyThreadState_GET(); c->_cs_mutex = m; c->_cs_prev = tstate->critical_section; tstate->critical_section = (uintptr_t)c; } else { - _PyCriticalSection_BeginSlow(c, m); + _PyCriticalSection_BeginSlow(tstate, c, m); } } -#define PyCriticalSection_BeginMutex _PyCriticalSection_BeginMutex static inline void -_PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) +_PyCriticalSection_Begin(PyThreadState *tstate, PyCriticalSection *c, PyObject *op) { - _PyCriticalSection_BeginMutex(c, &op->ob_mutex); + _PyCriticalSection_BeginMutex(tstate, c, &op->ob_mutex); } -#define PyCriticalSection_Begin _PyCriticalSection_Begin // Removes the top-most critical section from the thread's stack of critical // sections. If the new top-most critical section is inactive, then it is // resumed. static inline void -_PyCriticalSection_Pop(PyCriticalSection *c) +_PyCriticalSection_Pop(PyThreadState *tstate, PyCriticalSection *c) { - PyThreadState *tstate = _PyThreadState_GET(); uintptr_t prev = c->_cs_prev; tstate->critical_section = prev; @@ -132,7 +137,7 @@ _PyCriticalSection_Pop(PyCriticalSection *c) } static inline void -_PyCriticalSection_End(PyCriticalSection *c) +_PyCriticalSection_End(PyThreadState *tstate, PyCriticalSection *c) { // If the mutex is NULL, we used the fast path in // _PyCriticalSection_BeginSlow for locks already held in the top-most @@ -141,18 +146,17 @@ _PyCriticalSection_End(PyCriticalSection *c) return; } PyMutex_Unlock(c->_cs_mutex); - _PyCriticalSection_Pop(c); + _PyCriticalSection_Pop(tstate, c); } -#define PyCriticalSection_End _PyCriticalSection_End static inline void -_PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) +_PyCriticalSection2_BeginMutex(PyThreadState *tstate, PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) { if (m1 == m2) { // If the two mutex arguments are the same, treat this as a critical // section with a single mutex. c->_cs_mutex2 = NULL; - _PyCriticalSection_BeginMutex(&c->_cs_base, m1); + _PyCriticalSection_BeginMutex(tstate, &c->_cs_base, m1); return; } @@ -167,7 +171,6 @@ _PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) if (PyMutex_LockFast(m1)) { if (PyMutex_LockFast(m2)) { - PyThreadState *tstate = _PyThreadState_GET(); c->_cs_base._cs_mutex = m1; c->_cs_mutex2 = m2; c->_cs_base._cs_prev = tstate->critical_section; @@ -176,24 +179,22 @@ _PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) tstate->critical_section = p; } else { - _PyCriticalSection2_BeginSlow(c, m1, m2, 1); + _PyCriticalSection2_BeginSlow(tstate, c, m1, m2, 1); } } else { - _PyCriticalSection2_BeginSlow(c, m1, m2, 0); + _PyCriticalSection2_BeginSlow(tstate, c, m1, m2, 0); } } -#define PyCriticalSection2_BeginMutex _PyCriticalSection2_BeginMutex static inline void -_PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) +_PyCriticalSection2_Begin(PyThreadState *tstate, PyCriticalSection2 *c, PyObject *a, PyObject *b) { - _PyCriticalSection2_BeginMutex(c, &a->ob_mutex, &b->ob_mutex); + _PyCriticalSection2_BeginMutex(tstate, c, &a->ob_mutex, &b->ob_mutex); } -#define PyCriticalSection2_Begin _PyCriticalSection2_Begin static inline void -_PyCriticalSection2_End(PyCriticalSection2 *c) +_PyCriticalSection2_End(PyThreadState *tstate, PyCriticalSection2 *c) { // if mutex1 is NULL, we used the fast path in // _PyCriticalSection_BeginSlow for mutexes that are already held, @@ -207,9 +208,8 @@ _PyCriticalSection2_End(PyCriticalSection2 *c) PyMutex_Unlock(c->_cs_mutex2); } PyMutex_Unlock(c->_cs_base._cs_mutex); - _PyCriticalSection_Pop(&c->_cs_base); + _PyCriticalSection_Pop(tstate, &c->_cs_base); } -#define PyCriticalSection2_End _PyCriticalSection2_End static inline void _PyCriticalSection_AssertHeld(PyMutex *mutex) @@ -251,6 +251,45 @@ _PyCriticalSection_AssertHeldObj(PyObject *op) #endif } + +#undef Py_BEGIN_CRITICAL_SECTION +# define Py_BEGIN_CRITICAL_SECTION(op) \ + { \ + PyCriticalSection _py_cs; \ + PyThreadState *_cs_tstate = _PyThreadState_GET(); \ + _PyCriticalSection_Begin(_cs_tstate, &_py_cs, _PyObject_CAST(op)) + +#undef Py_BEGIN_CRITICAL_SECTION_MUTEX +# define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ + { \ + PyCriticalSection _py_cs; \ + PyThreadState *_cs_tstate = _PyThreadState_GET(); \ + _PyCriticalSection_BeginMutex(_cs_tstate, &_py_cs, mutex) + +#undef Py_END_CRITICAL_SECTION +# define Py_END_CRITICAL_SECTION() \ + _PyCriticalSection_End(_cs_tstate, &_py_cs); \ + } + +#undef Py_BEGIN_CRITICAL_SECTION2 +# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyThreadState *_cs_tstate = _PyThreadState_GET(); \ + _PyCriticalSection2_Begin(_cs_tstate, &_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) + +#undef Py_BEGIN_CRITICAL_SECTION2_MUTEX +# define Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyThreadState *_cs_tstate = _PyThreadState_GET(); \ + _PyCriticalSection2_BeginMutex(_cs_tstate, &_py_cs2, m1, m2) + +#undef Py_END_CRITICAL_SECTION2 +# define Py_END_CRITICAL_SECTION2() \ + _PyCriticalSection2_End(_cs_tstate, &_py_cs2); \ + } + #endif /* Py_GIL_DISABLED */ #ifdef __cplusplus diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 81faffac194..bed966681fa 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -265,6 +265,12 @@ typedef struct { // heap types PyObject *PyExc_NotShareableError; } exceptions; + + // Cached references to pickle.dumps/loads (per-interpreter). + struct { + PyObject *dumps; + PyObject *loads; + } pickle; } _PyXI_state_t; #define _PyXI_GET_GLOBAL_STATE(interp) (&(interp)->runtime->xi) diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 8e7cd16acff..1dd10f8d94c 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -102,12 +102,23 @@ typedef struct _Py_DebugOffsets { uint64_t next; uint64_t interp; uint64_t current_frame; + uint64_t base_frame; + uint64_t last_profiled_frame; uint64_t thread_id; uint64_t native_thread_id; uint64_t datastack_chunk; uint64_t status; + uint64_t holds_gil; + uint64_t gil_requested; + uint64_t current_exception; + uint64_t exc_state; } thread_state; + // Exception stack item offset + struct { + uint64_t exc_value; + } err_stackitem; + // InterpreterFrame offset; struct _interpreter_frame { uint64_t size; @@ -204,12 +215,16 @@ typedef struct _Py_DebugOffsets { uint64_t state; uint64_t length; uint64_t asciiobject_size; + uint64_t compactunicodeobject_size; } unicode_object; // GC runtime state offset; struct _gc { uint64_t size; uint64_t collecting; + uint64_t frame; + uint64_t generation_stats_size; + uint64_t generation_stats; } gc; // Generator object offset; @@ -269,10 +284,19 @@ typedef struct _Py_DebugOffsets { .next = offsetof(PyThreadState, next), \ .interp = offsetof(PyThreadState, interp), \ .current_frame = offsetof(PyThreadState, current_frame), \ + .base_frame = offsetof(PyThreadState, base_frame), \ + .last_profiled_frame = offsetof(PyThreadState, last_profiled_frame), \ .thread_id = offsetof(PyThreadState, thread_id), \ .native_thread_id = offsetof(PyThreadState, native_thread_id), \ .datastack_chunk = offsetof(PyThreadState, datastack_chunk), \ .status = offsetof(PyThreadState, _status), \ + .holds_gil = offsetof(PyThreadState, holds_gil), \ + .gil_requested = offsetof(PyThreadState, gil_requested), \ + .current_exception = offsetof(PyThreadState, current_exception), \ + .exc_state = offsetof(PyThreadState, exc_state), \ + }, \ + .err_stackitem = { \ + .exc_value = offsetof(_PyErr_StackItem, exc_value), \ }, \ .interpreter_frame = { \ .size = sizeof(_PyInterpreterFrame), \ @@ -347,10 +371,14 @@ typedef struct _Py_DebugOffsets { .state = offsetof(PyUnicodeObject, _base._base.state), \ .length = offsetof(PyUnicodeObject, _base._base.length), \ .asciiobject_size = sizeof(PyASCIIObject), \ + .compactunicodeobject_size = sizeof(PyCompactUnicodeObject), \ }, \ .gc = { \ .size = sizeof(struct _gc_runtime_state), \ .collecting = offsetof(struct _gc_runtime_state, collecting), \ + .frame = offsetof(struct _gc_runtime_state, frame), \ + .generation_stats_size = sizeof(struct gc_stats), \ + .generation_stats = offsetof(struct _gc_runtime_state, generation_stats), \ }, \ .gen_object = { \ .size = sizeof(PyGenObject), \ diff --git a/Include/internal/pycore_descrobject.h b/Include/internal/pycore_descrobject.h index 3cec59a68a3..6143f82176a 100644 --- a/Include/internal/pycore_descrobject.h +++ b/Include/internal/pycore_descrobject.h @@ -22,6 +22,8 @@ typedef propertyobject _PyPropertyObject; extern PyTypeObject _PyMethodWrapper_Type; +extern void *_PyMember_GetOffset(PyObject *, PyMemberDef *); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 6ab569393e5..6c6e3b77e69 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -30,14 +30,14 @@ PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyDict_DelItem_KnownHash(PyObject *mp, PyObject *key, Py_hash_t hash); + +// Exported for external JIT support +PyAPI_FUNC(int) _PyDict_DelItem_KnownHash_LockHeld(PyObject *mp, PyObject *key, + Py_hash_t hash); + extern int _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); -// "Id" variants -extern PyObject* _PyDict_GetItemIdWithError(PyObject *dp, - _Py_Identifier *key); -extern int _PyDict_ContainsId(PyObject *, _Py_Identifier *); -extern int _PyDict_SetItemId(PyObject *dp, _Py_Identifier *key, PyObject *item); -extern int _PyDict_DelItemId(PyObject *mp, _Py_Identifier *key); +extern void _PyDict_ClearKeysVersionLockHeld(PyObject *mp); extern int _PyDict_Next( PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash); @@ -47,6 +47,8 @@ extern int _PyDict_HasOnlyStringKeys(PyObject *mp); // Export for '_ctypes' shared extension PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); +extern Py_ssize_t _PyDict_SizeOf_LockHeld(PyDictObject *); + #define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) /* Like PyDict_Merge, but override can be 0, 1 or 2. If override is 0, @@ -54,7 +56,7 @@ PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); of a key wins, if override is 2, a KeyError with conflicting key as argument is raised. */ -PyAPI_FUNC(int) _PyDict_MergeEx(PyObject *mp, PyObject *other, int override); +PyAPI_FUNC(int) _PyDict_MergeUniq(PyObject *mp, PyObject *other, PyObject **dupkey); extern void _PyDict_DebugMallocStats(FILE *out); @@ -86,9 +88,15 @@ typedef struct { extern PyDictKeysObject *_PyDict_NewKeysForClass(PyHeapTypeObject *); extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); +/* Implementations of the `|` and `|=` operators for dict, used by the + * specializing interpreter. */ +extern PyObject *_PyDict_Or(PyObject *self, PyObject *other); +extern PyObject *_PyDict_IOr(PyObject *self, PyObject *other); + /* Gets a version number unique to the current state of the keys of dict, if possible. - * Returns the version number, or zero if it was not possible to get a version number. */ -extern uint32_t _PyDictKeys_GetVersionForCurrentState( + * Returns the version number, or zero if it was not possible to get a version number. + * Exported for external JIT support */ +PyAPI_FUNC(uint32_t) _PyDictKeys_GetVersionForCurrentState( PyInterpreterState *interp, PyDictKeysObject *dictkeys); /* Gets a version number unique to the current state of the keys of dict, if possible. @@ -98,8 +106,9 @@ extern uint32_t _PyDictKeys_GetVersionForCurrentState( * * The caller must hold the per-object lock on dict. * - * Returns the version number, or zero if it was not possible to get a version number. */ -extern uint32_t _PyDict_GetKeysVersionForCurrentState( + * Returns the version number, or zero if it was not possible to get a version number. + * Exported for external JIT support */ +PyAPI_FUNC(uint32_t) _PyDict_GetKeysVersionForCurrentState( PyInterpreterState *interp, PyDictObject *dict); extern size_t _PyDict_KeysSize(PyDictKeysObject *keys); @@ -108,15 +117,18 @@ extern void _PyDictKeys_DecRef(PyDictKeysObject *keys); /* _Py_dict_lookup() returns index of entry which can be used like DK_ENTRIES(dk)[index]. * -1 when no entry found, -3 when compare raises error. + * Exported for external JIT support */ -extern Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); +PyAPI_FUNC(Py_ssize_t) _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); extern Py_ssize_t _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); extern Py_ssize_t _Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject *key, Py_hash_t hash, _PyStackRef *value_addr); extern int _PyDict_GetMethodStackRef(PyDictObject *dict, PyObject *name, _PyStackRef *method); -extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *); -extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); +// Exported for external JIT support +PyAPI_FUNC(Py_ssize_t) _PyDict_LookupIndexAndValue(PyDictObject *, PyObject *, PyObject **); +PyAPI_FUNC(Py_ssize_t) _PyDict_LookupIndex(PyDictObject *, PyObject *); +PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); /* Look up a string key in an all unicode dict keys, assign the keys object a version, and * store it in version. @@ -125,9 +137,11 @@ extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject * strings. * * Returns DKIX_EMPTY if the key is not present. + * + * Exported for external JIT support */ -extern Py_ssize_t _PyDictKeys_StringLookupAndVersion(PyDictKeysObject* dictkeys, PyObject *key, uint32_t *version); -extern Py_ssize_t _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys, PyObject *key); +PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookupAndVersion(PyDictKeysObject* dictkeys, PyObject *key, uint32_t *version); +PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys, PyObject *key); PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); PyAPI_FUNC(void) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *, PyObject *, _PyStackRef *); @@ -136,15 +150,17 @@ extern PyObject *_PyDict_LoadBuiltinsFromGlobals(PyObject *globals); /* Consumes references to key and value */ PyAPI_FUNC(int) _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); -extern int _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value); +PyAPI_FUNC(int) _PyDict_SetItem_Take2_KnownHash(PyDictObject *op, PyObject *key, PyObject *value, Py_hash_t hash); +// Exported for external JIT support +PyAPI_FUNC(int) _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyDict_SetItem_KnownHash_LockHeld(PyDictObject *mp, PyObject *key, PyObject *value, Py_hash_t hash); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyDict_GetItemRef_KnownHash_LockHeld(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); -extern int _PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); +PyAPI_FUNC(int) _PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); extern int _PyDict_GetItemRef_Unicode_LockHeld(PyDictObject *op, PyObject *key, PyObject **result); -extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *name, PyObject *value); +PyAPI_FUNC(int) _PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *name, PyObject *value); extern int _PyDict_Pop_KnownHash( PyDictObject *dict, @@ -158,6 +174,9 @@ extern void _PyDict_Clear_LockHeld(PyObject *op); PyAPI_FUNC(void) _PyDict_EnsureSharedOnRead(PyDictObject *mp); #endif +// Export for '_elementtree' shared extension +PyAPI_FUNC(PyObject*) _PyDict_CopyAsDict(PyObject *op); + #define DKIX_EMPTY (-1) #define DKIX_DUMMY (-2) /* Used internally */ #define DKIX_ERROR (-3) @@ -264,6 +283,13 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DICT_UNIQUE_ID_SHIFT (32) #define DICT_UNIQUE_ID_MAX ((UINT64_C(1) << (64 - DICT_UNIQUE_ID_SHIFT)) - 1) +/* The first three dict watcher IDs are reserved for CPython, + * so we don't need to check that they haven't been used */ +#define BUILTINS_WATCHER_ID 0 +#define GLOBALS_WATCHER_ID 1 +#define MODULE_WATCHER_ID 2 +#define FIRST_AVAILABLE_WATCHER 3 + PyAPI_FUNC(void) _PyDict_SendEvent(int watcher_bits, @@ -273,14 +299,13 @@ _PyDict_SendEvent(int watcher_bits, PyObject *value); static inline void -_PyDict_NotifyEvent(PyInterpreterState *interp, - PyDict_WatchEvent event, +_PyDict_NotifyEvent(PyDict_WatchEvent event, PyDictObject *mp, PyObject *key, PyObject *value) { assert(Py_REFCNT((PyObject*)mp) > 0); - int watcher_bits = mp->_ma_watcher_tag & DICT_WATCHER_MASK; + int watcher_bits = FT_ATOMIC_LOAD_UINT64_ACQUIRE(mp->_ma_watcher_tag) & DICT_WATCHER_MASK; if (watcher_bits) { RARE_EVENT_STAT_INC(watched_dict_modification); _PyDict_SendEvent(watcher_bits, event, mp, key, value); @@ -312,6 +337,10 @@ _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix) values->size = size+1; } +// Exported for external JIT support +PyAPI_FUNC(void) +_PyDict_InsertSplitValue(PyDictObject *mp, PyObject *key, PyObject *value, Py_ssize_t ix); + static inline size_t shared_keys_usable_size(PyDictKeysObject *keys) { @@ -356,13 +385,13 @@ PyDictObject *_PyObject_MaterializeManagedDict_LockHeld(PyObject *); static inline Py_ssize_t _PyDict_UniqueId(PyDictObject *mp) { - return (Py_ssize_t)(mp->_ma_watcher_tag >> DICT_UNIQUE_ID_SHIFT); + return (Py_ssize_t)(FT_ATOMIC_LOAD_UINT64_RELAXED(mp->_ma_watcher_tag) >> DICT_UNIQUE_ID_SHIFT); } static inline void _Py_INCREF_DICT(PyObject *op) { - assert(PyDict_Check(op)); + assert(PyAnyDict_Check(op)); Py_ssize_t id = _PyDict_UniqueId((PyDictObject *)op); _Py_THREAD_INCREF_OBJECT(op, id); } @@ -370,7 +399,7 @@ _Py_INCREF_DICT(PyObject *op) static inline void _Py_DECREF_DICT(PyObject *op) { - assert(PyDict_Check(op)); + assert(PyAnyDict_Check(op)); Py_ssize_t id = _PyDict_UniqueId((PyDictObject *)op); _Py_THREAD_DECREF_OBJECT(op, id); } @@ -400,6 +429,15 @@ _Py_DECREF_BUILTINS(PyObject *op) } #endif +/* frozendict */ +typedef struct { + PyDictObject ob_base; + Py_hash_t ma_hash; +} PyFrozenDictObject; + +#define _PyFrozenDictObject_CAST(op) \ + (assert(PyFrozenDict_Check(op)), _Py_CAST(PyFrozenDictObject*, (op))) + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_emscripten_trampoline.h b/Include/internal/pycore_emscripten_trampoline.h index 16916f1a8eb..e37c53a64f4 100644 --- a/Include/internal/pycore_emscripten_trampoline.h +++ b/Include/internal/pycore_emscripten_trampoline.h @@ -27,9 +27,6 @@ #if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) -void -_Py_EmscriptenTrampoline_Init(_PyRuntimeState *runtime); - PyObject* _PyEM_TrampolineCall(PyCFunctionWithKeywords func, PyObject* self, diff --git a/Include/internal/pycore_faulthandler.h b/Include/internal/pycore_faulthandler.h index 78cd657e6ae..9ddd70d39ed 100644 --- a/Include/internal/pycore_faulthandler.h +++ b/Include/internal/pycore_faulthandler.h @@ -42,6 +42,7 @@ struct faulthandler_user_signal { int chain; _Py_sighandler_t previous; PyInterpreterState *interp; + Py_ssize_t max_threads; }; #endif /* FAULTHANDLER_USER */ @@ -57,6 +58,7 @@ struct _faulthandler_runtime_state { void *exc_handler; #endif int c_stack; + Py_ssize_t max_threads; } fatal_error; struct { @@ -68,6 +70,7 @@ struct _faulthandler_runtime_state { int exit; char *header; size_t header_len; + Py_ssize_t max_threads; /* The main thread always holds this lock. It is only released when faulthandler_thread() is interrupted before this thread exits, or at Python exit. */ diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index 317f984188b..62501cdaf44 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -12,7 +12,6 @@ extern "C" { /* runtime lifecycle */ -extern void _PyFloat_InitState(PyInterpreterState *); extern PyStatus _PyFloat_InitTypes(PyInterpreterState *); extern void _PyFloat_FiniType(PyInterpreterState *); @@ -42,6 +41,15 @@ extern double _Py_parse_inf_or_nan(const char *p, char **endptr); extern int _Py_convert_int_to_double(PyObject **v, double *dbl); +/* Should match endianness of the platform in most (all?) cases. */ + +#ifdef DOUBLE_IS_BIG_ENDIAN_IEEE754 +# define _PY_FLOAT_BIG_ENDIAN 1 +# define _PY_FLOAT_LITTLE_ENDIAN 0 +#else +# define _PY_FLOAT_BIG_ENDIAN 0 +# define _PY_FLOAT_LITTLE_ENDIAN 1 +#endif #ifdef __cplusplus } diff --git a/Include/internal/pycore_flowgraph.h b/Include/internal/pycore_flowgraph.h index 5043260d2fd..fc3ae6b369c 100644 --- a/Include/internal/pycore_flowgraph.h +++ b/Include/internal/pycore_flowgraph.h @@ -27,7 +27,7 @@ int _PyCfg_OptimizeCodeUnit(struct _PyCfgBuilder *g, PyObject *consts, PyObject struct _PyCfgBuilder* _PyCfg_FromInstructionSequence(_PyInstructionSequence *seq); int _PyCfg_ToInstructionSequence(struct _PyCfgBuilder *g, _PyInstructionSequence *seq); int _PyCfg_OptimizedCfgToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_CodeUnitMetadata *umd, - int code_flags, int *stackdepth, int *nlocalsplus, + int *stackdepth, int *nlocalsplus, _PyInstructionSequence *seq); PyCodeObject * diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 8c410e9e208..3c9ab99c34e 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -38,23 +38,23 @@ struct _frame { PyObject *_f_frame_data[1]; }; -extern PyFrameObject* _PyFrame_New_NoTrack(PyCodeObject *code); +// Exported for external JIT support +PyAPI_FUNC(PyFrameObject *) _PyFrame_New_NoTrack(PyCodeObject *code); /* other API */ typedef enum _framestate { - FRAME_CREATED = -3, - FRAME_SUSPENDED = -2, - FRAME_SUSPENDED_YIELD_FROM = -1, - FRAME_EXECUTING = 0, - FRAME_COMPLETED = 1, - FRAME_CLEARED = 4 + FRAME_CREATED = 0, + FRAME_SUSPENDED = 1, + FRAME_SUSPENDED_YIELD_FROM = 2, + FRAME_SUSPENDED_YIELD_FROM_LOCKED = 3, + FRAME_EXECUTING = 4, + FRAME_CLEARED = 5 } PyFrameState; -#define FRAME_STATE_SUSPENDED(S) ((S) == FRAME_SUSPENDED || (S) == FRAME_SUSPENDED_YIELD_FROM) -#define FRAME_STATE_FINISHED(S) ((S) >= FRAME_COMPLETED) - +#define FRAME_STATE_SUSPENDED(S) ((S) >= FRAME_SUSPENDED && (S) <= FRAME_SUSPENDED_YIELD_FROM_LOCKED) +#define FRAME_STATE_FINISHED(S) ((S) == FRAME_CLEARED) #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h index f3c9a669ad3..3a41ec4b54b 100644 --- a/Include/internal/pycore_freelist.h +++ b/Include/internal/pycore_freelist.h @@ -17,15 +17,16 @@ extern "C" { static inline struct _Py_freelists * _Py_freelists_GET(void) { - PyThreadState *tstate = _PyThreadState_GET(); #ifdef Py_DEBUG - _Py_EnsureTstateNotNULL(tstate); + _Py_AssertHoldsTstate(); #endif #ifdef Py_GIL_DISABLED + PyThreadState *tstate = _PyThreadState_GET(); return &((_PyThreadStateImpl*)tstate)->freelists; #else - return &tstate->interp->object_state.freelists; + PyInterpreterState *interp = _PyInterpreterState_GET(); + return &interp->object_state.freelists; #endif } diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h index 59beb92f3f7..46e2a82ea03 100644 --- a/Include/internal/pycore_freelist_state.h +++ b/Include/internal/pycore_freelist_state.h @@ -27,6 +27,7 @@ extern "C" { # define Py_futureiters_MAXFREELIST 255 # define Py_object_stack_chunks_MAXFREELIST 4 # define Py_unicode_writers_MAXFREELIST 1 +# define Py_bytes_writers_MAXFREELIST 1 # define Py_pycfunctionobject_MAXFREELIST 16 # define Py_pycmethodobject_MAXFREELIST 16 # define Py_pymethodobjects_MAXFREELIST 20 @@ -61,6 +62,7 @@ struct _Py_freelists { struct _Py_freelist futureiters; struct _Py_freelist object_stack_chunks; struct _Py_freelist unicode_writers; + struct _Py_freelist bytes_writers; struct _Py_freelist pycfunctionobject; struct _Py_freelist pycmethodobject; struct _Py_freelist pymethodobjects; diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 6e120965956..2184f40956d 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -extern PyObject* _PyFunction_Vectorcall( +PyAPI_FUNC(PyObject *) _PyFunction_Vectorcall( PyObject *func, PyObject *const *stack, size_t nargsf, @@ -27,10 +27,10 @@ _PyFunction_IsVersionValid(uint32_t version) return version >= FUNC_VERSION_FIRST_VALID; } -extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); +// Exported for external JIT support +PyAPI_FUNC(uint32_t) _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); PyAPI_FUNC(void) _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version); void _PyFunction_ClearCodeByVersion(uint32_t version); -PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version, PyObject **p_code); extern PyObject *_Py_set_function_type_params( PyThreadState* unused, PyObject *func, PyObject *type_params); @@ -47,6 +47,17 @@ static inline PyObject* _PyFunction_GET_BUILTINS(PyObject *func) { #define _PyFunction_GET_BUILTINS(func) _PyFunction_GET_BUILTINS(_PyObject_CAST(func)) +/* Get the callable wrapped by a classmethod. + Returns a borrowed reference. + The caller must ensure 'cm' is a classmethod object. */ +extern PyObject *_PyClassMethod_GetFunc(PyObject *cm); + +/* Get the callable wrapped by a staticmethod. + Returns a borrowed reference. + The caller must ensure 'sm' is a staticmethod object. */ +extern PyObject *_PyStaticMethod_GetFunc(PyObject *sm); + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index a6519aa0863..e105677cd2e 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -118,21 +118,6 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) { /* Bit 1 is set when the object is in generation which is GCed currently. */ #define _PyGC_PREV_MASK_COLLECTING ((uintptr_t)2) -/* Bit 0 in _gc_next is the old space bit. - * It is set as follows: - * Young: gcstate->visited_space - * old[0]: 0 - * old[1]: 1 - * permanent: 0 - * - * During a collection all objects handled should have the bit set to - * gcstate->visited_space, as objects are moved from the young gen - * and the increment into old[gcstate->visited_space]. - * When object are moved from the pending space, old[gcstate->visited_space^1] - * into the increment, the old space bit is flipped. -*/ -#define _PyGC_NEXT_MASK_OLD_SPACE_1 1 - #define _PyGC_PREV_SHIFT 2 #define _PyGC_PREV_MASK (((uintptr_t) -1) << _PyGC_PREV_SHIFT) @@ -159,13 +144,11 @@ typedef enum { // Lowest bit of _gc_next is used for flags only in GC. // But it is always 0 for normal code. static inline PyGC_Head* _PyGCHead_NEXT(PyGC_Head *gc) { - uintptr_t next = gc->_gc_next & _PyGC_PREV_MASK; + uintptr_t next = gc->_gc_next; return (PyGC_Head*)next; } static inline void _PyGCHead_SET_NEXT(PyGC_Head *gc, PyGC_Head *next) { - uintptr_t unext = (uintptr_t)next; - assert((unext & ~_PyGC_PREV_MASK) == 0); - gc->_gc_next = (gc->_gc_next & ~_PyGC_PREV_MASK) | unext; + gc->_gc_next = (uintptr_t)next; } // Lowest two bits of _gc_prev is used for _PyGC_PREV_MASK_* flags. @@ -205,6 +188,8 @@ static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { #endif } +extern void _Py_ScheduleGC(PyThreadState *tstate); + /* Tell the GC to track this object. * @@ -214,7 +199,7 @@ static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { * ob_traverse method. * * Internal note: interp->gc.generation0->_gc_prev doesn't have any bit flags - * because it's not object header. So we don't use _PyGCHead_PREV() and + * because it's not an object header. So we don't use _PyGCHead_PREV() and * _PyGCHead_SET_PREV() for it to avoid unnecessary bitwise operations. * * See also the public PyObject_GC_Track() function. @@ -238,13 +223,11 @@ static inline void _PyObject_GC_TRACK( "object is in generation which is garbage collected", filename, lineno, __func__); - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyGC_Head *generation0 = &interp->gc.young.head; + PyGC_Head *generation0 = _PyInterpreterState_GET()->gc.generation0; PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); - uintptr_t not_visited = 1 ^ interp->gc.visited_space; - gc->_gc_next = ((uintptr_t)generation0) | not_visited; + _PyGCHead_SET_NEXT(gc, generation0); generation0->_gc_prev = (uintptr_t)gc; #endif } @@ -343,7 +326,6 @@ extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs); // Functions to clear types free lists extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); -extern void _Py_ScheduleGC(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); union _PyStackRef; diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index c1fc3511f84..2c264c39ae9 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -22,7 +22,7 @@ PyGenObject *_PyGen_GetGeneratorFromFrame(_PyInterpreterFrame *frame) } PyAPI_FUNC(PyObject *)_PyGen_yf(PyGenObject *); -extern void _PyGen_Finalize(PyObject *self); +extern int _PyGen_ClearFrame(PyGenObject *self); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); @@ -31,7 +31,10 @@ PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); PyAPI_FUNC(PyObject *)_PyCoro_GetAwaitableIter(PyObject *o); -extern PyObject *_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); +PyAPI_FUNC(PyObject *)_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); + +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyCoro_ComputeOrigin(int origin_depth, _PyInterpreterFrame *current_frame); extern PyTypeObject _PyCoroWrapper_Type; extern PyTypeObject _PyAsyncGenWrappedValue_Type; diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 63888eab7b4..f7d3dcd440a 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -13,7 +13,7 @@ static inline void _PyStaticObject_CheckRefcnt(PyObject *obj) { if (!_Py_IsImmortal(obj)) { fprintf(stderr, "Immortal Object has less refcnt than expected.\n"); - _PyObject_Dump(obj); + PyObject_Dump(obj); } } #endif @@ -286,6 +286,774 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 254]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 255]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 256]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 257]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 258]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 259]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 260]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 261]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 262]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 263]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 264]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 265]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 266]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 267]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 268]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 269]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 270]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 271]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 272]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 273]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 274]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 275]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 276]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 277]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 278]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 279]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 280]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 281]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 282]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 283]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 284]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 285]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 286]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 287]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 288]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 289]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 290]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 291]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 292]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 293]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 294]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 295]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 296]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 297]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 298]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 299]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 300]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 301]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 302]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 303]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 304]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 305]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 306]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 307]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 308]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 309]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 310]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 311]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 312]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 313]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 314]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 315]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 316]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 317]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 318]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 319]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 320]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 321]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 322]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 323]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 324]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 325]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 326]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 327]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 328]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 329]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 330]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 331]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 332]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 333]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 334]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 335]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 336]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 337]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 338]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 339]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 340]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 341]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 342]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 343]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 344]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 345]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 346]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 347]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 348]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 349]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 350]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 351]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 352]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 353]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 354]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 355]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 356]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 357]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 358]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 359]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 360]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 361]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 362]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 363]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 364]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 365]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 366]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 367]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 368]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 369]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 370]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 371]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 372]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 373]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 374]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 375]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 376]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 377]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 378]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 379]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 380]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 381]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 382]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 383]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 384]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 385]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 386]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 387]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 388]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 389]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 390]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 391]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 392]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 393]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 394]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 395]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 396]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 397]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 398]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 399]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 400]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 401]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 402]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 403]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 404]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 405]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 406]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 407]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 408]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 409]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 410]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 411]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 412]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 413]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 414]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 415]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 416]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 417]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 418]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 419]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 420]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 421]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 422]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 423]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 424]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 425]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 426]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 427]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 428]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 429]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 430]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 431]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 432]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 433]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 434]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 435]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 436]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 437]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 438]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 439]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 440]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 441]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 442]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 443]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 444]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 445]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 446]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 447]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 448]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 449]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 450]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 451]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 452]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 453]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 454]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 455]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 456]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 457]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 458]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 459]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 460]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 461]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 462]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 463]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 464]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 465]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 466]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 467]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 468]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 469]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 470]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 471]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 472]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 473]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 474]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 475]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 476]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 477]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 478]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 479]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 480]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 481]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 482]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 483]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 484]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 485]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 486]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 487]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 488]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 489]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 490]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 491]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 492]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 493]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 494]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 495]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 496]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 497]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 498]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 499]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 500]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 501]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 502]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 503]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 504]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 505]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 506]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 507]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 508]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 509]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 510]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 511]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 512]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 513]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 514]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 515]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 516]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 517]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 518]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 519]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 520]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 521]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 522]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 523]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 524]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 525]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 526]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 527]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 528]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 529]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 530]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 531]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 532]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 533]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 534]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 535]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 536]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 537]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 538]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 539]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 540]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 541]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 542]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 543]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 544]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 545]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 546]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 547]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 548]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 549]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 550]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 551]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 552]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 553]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 554]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 555]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 556]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 557]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 558]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 559]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 560]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 561]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 562]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 563]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 564]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 565]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 566]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 567]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 568]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 569]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 570]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 571]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 572]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 573]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 574]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 575]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 576]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 577]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 578]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 579]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 580]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 581]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 582]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 583]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 584]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 585]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 586]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 587]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 588]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 589]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 590]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 591]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 592]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 593]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 594]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 595]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 596]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 597]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 598]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 599]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 600]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 601]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 602]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 603]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 604]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 605]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 606]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 607]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 608]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 609]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 610]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 611]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 612]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 613]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 614]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 615]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 616]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 617]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 618]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 619]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 620]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 621]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 622]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 623]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 624]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 625]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 626]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 627]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 628]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 629]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 630]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 631]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 632]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 633]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 634]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 635]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 636]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 637]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 638]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 639]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 640]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 641]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 642]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 643]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 644]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 645]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 646]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 647]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 648]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 649]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 650]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 651]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 652]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 653]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 654]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 655]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 656]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 657]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 658]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 659]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 660]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 661]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 662]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 663]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 664]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 665]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 666]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 667]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 668]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 669]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 670]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 671]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 672]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 673]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 674]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 675]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 676]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 677]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 678]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 679]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 680]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 681]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 682]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 683]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 684]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 685]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 686]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 687]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 688]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 689]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 690]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 691]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 692]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 693]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 694]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 695]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 696]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 697]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 698]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 699]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 700]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 701]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 702]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 703]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 704]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 705]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 706]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 707]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 708]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 709]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 710]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 711]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 712]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 713]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 714]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 715]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 716]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 717]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 718]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 719]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 720]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 721]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 722]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 723]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 724]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 725]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 726]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 727]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 728]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 729]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 730]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 731]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 732]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 733]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 734]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 735]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 736]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 737]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 738]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 739]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 740]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 741]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 742]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 743]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 744]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 745]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 746]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 747]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 748]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 749]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 750]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 751]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 752]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 753]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 754]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 755]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 756]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 757]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 758]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 759]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 760]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 761]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 762]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 763]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 764]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 765]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 766]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 767]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 768]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 769]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 770]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 771]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 772]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 773]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 774]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 775]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 776]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 777]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 778]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 779]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 780]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 781]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 782]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 783]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 784]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 785]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 786]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 787]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 788]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 789]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 790]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 791]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 792]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 793]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 794]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 795]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 796]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 797]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 798]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 799]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 800]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 801]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 802]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 803]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 804]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 805]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 806]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 807]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 808]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 809]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 810]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 811]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 812]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 813]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 814]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 815]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 816]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 817]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 818]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 819]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 820]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 821]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 822]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 823]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 824]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 825]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 826]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 827]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 828]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 829]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 830]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 831]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 832]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 833]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 834]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 835]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 836]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 837]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 838]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 839]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 840]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 841]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 842]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 843]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 844]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 845]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 846]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 847]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 848]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 849]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 850]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 851]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 852]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 853]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 854]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 855]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 856]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 857]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 858]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 859]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 860]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 861]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 862]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 863]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 864]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 865]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 866]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 867]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 868]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 869]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 870]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 871]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 872]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 873]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 874]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 875]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 876]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 877]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 878]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 879]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 880]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 881]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 882]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 883]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 884]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 885]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 886]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 887]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 888]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 889]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 890]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 891]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 892]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 893]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 894]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 895]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 896]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 897]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 898]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 899]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 900]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 901]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 902]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 903]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 904]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 905]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 906]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 907]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 908]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 909]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 910]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 911]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 912]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 913]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 914]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 915]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 916]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 917]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 918]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 919]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 920]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 921]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 922]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 923]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 924]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 925]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 926]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 927]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 928]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 929]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 930]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 931]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 932]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 933]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 934]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 935]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 936]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 937]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 938]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 939]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 940]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 941]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 942]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 943]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 944]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 945]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 946]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 947]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 948]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 949]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 950]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 951]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 952]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 953]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 954]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 955]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 956]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 957]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 958]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 959]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 960]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 961]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 962]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 963]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 964]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 965]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 966]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 967]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 968]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 969]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 970]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 971]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 972]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 973]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 974]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 975]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 976]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 977]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 978]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 979]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 980]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 981]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 982]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 983]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 984]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 985]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 986]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 987]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 988]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 989]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 990]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 991]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 992]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 993]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 994]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 995]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 996]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 997]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 998]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 999]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1000]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1001]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1002]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1003]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1004]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1005]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1006]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1007]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1008]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1009]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1010]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1011]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1012]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1013]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1014]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1015]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1016]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1017]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1018]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1019]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1020]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1021]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1022]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1023]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1024]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(bytes_characters)[0]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(bytes_characters)[1]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(bytes_characters)[2]); @@ -558,18 +1326,32 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot_locals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(empty)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(format)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(gc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(generic_base)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(list_err)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(native)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(str_replace_inf)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(utf_8)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(AGEN_CLOSED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(AGEN_CREATED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(AGEN_RUNNING)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(AGEN_SUSPENDED)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CANCELLED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CORO_CLOSED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CORO_CREATED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CORO_RUNNING)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CORO_SUSPENDED)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(Emax)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(Emin)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(FINISHED)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(False)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(GEN_CLOSED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(GEN_CREATED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(GEN_RUNNING)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(GEN_SUSPENDED)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(JSONDecodeError)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(PENDING)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(Py_Repr)); @@ -664,6 +1446,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__iter__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__itruediv__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__ixor__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__lazy_import__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__lazy_modules__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__le__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__len__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__length_hint__)); @@ -792,13 +1576,16 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aclose)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(add)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(add_done_callback)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(adobe)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_child)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(align)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all_interpreters)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all_threads)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(allow_code)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alphabet)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(any)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arg)); @@ -839,6 +1626,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_parameter_type)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_return)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_stack)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cache_frames)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_datetime_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_statements)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cadata)); @@ -849,6 +1637,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(callable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cancel)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(canonical)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(capath)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(capitals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(category)); @@ -882,9 +1671,11 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(co_varnames)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(col_offset)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(collector)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(command)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(comment_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(compile_mode)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(compression)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(config)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(consts)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(context)); @@ -933,6 +1724,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eager_start)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(effective_ids)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(element_factory)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(emptyerror)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(encode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(encoding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end)); @@ -946,7 +1738,9 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(event)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eventmask)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_tb)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_type)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_val)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(excepthook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exception)); @@ -958,6 +1752,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extra_tokens)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(facility)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(factory)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fallback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(false)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(family)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fanout)); @@ -980,6 +1775,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(flags)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(flush)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fold)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(foldspaces)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(follow_symlinks)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format_spec)); @@ -990,8 +1786,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromtimestamp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromutc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fset)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fullerror)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(func)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(future)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(gc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(generation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get_debug)); @@ -1018,6 +1816,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ident)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(identity_hint)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ignore)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ignorechars)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(imag)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(implieslink)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(importlib)); @@ -1096,12 +1895,15 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(loop)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(manual_reset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mapping)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mask)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(match)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_length)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_threads)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxdigits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxevents)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxlen)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxmem)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxsize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxsplit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxvalue)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(memLevel)); @@ -1133,6 +1935,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(name_from)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespace_separator)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespaces)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(native)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ndigits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nested)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_file_name)); @@ -1159,6 +1962,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(only_keys)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(oparg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(opcode)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(opcodes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(open)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(opener)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(operation)); @@ -1172,6 +1976,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outpath)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(overlapped)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(owner)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pad)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(padded)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pages)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(parameter)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(parent)); @@ -1195,12 +2001,16 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(print_file_and_line)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress_callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress_routine)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(proto)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(protocol)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ps1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ps2)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(qid)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(qualname)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(query)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(queuetype)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(quotetabs)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(raw)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(read)); @@ -1212,6 +2022,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(readline)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(readonly)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(real)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(recursive)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reducer_override)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(registry)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(rel_tol)); @@ -1231,6 +2042,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reversed)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(rounding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(salt)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sample_interval_us)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sched_priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(scheduler)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(script)); @@ -1261,6 +2073,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(size)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sizehint)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(skip_file_prefixes)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(skip_non_matching_threads)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sleep)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sock)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sort)); @@ -1269,9 +2082,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(spam)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(src)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(src_dir_fd)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stack_frames)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stacklevel)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(start)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(start_time_us)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(statement)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stats)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(status)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stderr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stdin)); @@ -1289,6 +2105,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(symmetric_difference_update)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tabsize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tag)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(take_bytes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(target)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(target_is_directory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(task)); @@ -1309,6 +2126,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(times)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timespec)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timestamp)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timestamp_us)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timetuple)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timeunit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(top)); @@ -1326,6 +2144,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tzinfo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tzname)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(uid)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(unboundop)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(unlink)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(unraisablehook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(updates)); @@ -1346,6 +2165,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(which)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(who)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(withdata)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wrapcol)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(writable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(write)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(write_through)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index b863a7c970e..22494b1798c 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -46,21 +46,35 @@ struct _Py_global_strings { STRUCT_FOR_STR(dot_locals, ".") STRUCT_FOR_STR(empty, "") STRUCT_FOR_STR(format, ".format") + STRUCT_FOR_STR(gc, "") STRUCT_FOR_STR(generic_base, ".generic_base") STRUCT_FOR_STR(json_decoder, "json.decoder") STRUCT_FOR_STR(kwdefaults, ".kwdefaults") STRUCT_FOR_STR(list_err, "list index out of range") + STRUCT_FOR_STR(native, "") STRUCT_FOR_STR(str_replace_inf, "1e309") STRUCT_FOR_STR(type_params, ".type_params") STRUCT_FOR_STR(utf_8, "utf-8") } literals; struct { + STRUCT_FOR_ID(AGEN_CLOSED) + STRUCT_FOR_ID(AGEN_CREATED) + STRUCT_FOR_ID(AGEN_RUNNING) + STRUCT_FOR_ID(AGEN_SUSPENDED) STRUCT_FOR_ID(CANCELLED) + STRUCT_FOR_ID(CORO_CLOSED) + STRUCT_FOR_ID(CORO_CREATED) + STRUCT_FOR_ID(CORO_RUNNING) + STRUCT_FOR_ID(CORO_SUSPENDED) STRUCT_FOR_ID(Emax) STRUCT_FOR_ID(Emin) STRUCT_FOR_ID(FINISHED) STRUCT_FOR_ID(False) + STRUCT_FOR_ID(GEN_CLOSED) + STRUCT_FOR_ID(GEN_CREATED) + STRUCT_FOR_ID(GEN_RUNNING) + STRUCT_FOR_ID(GEN_SUSPENDED) STRUCT_FOR_ID(JSONDecodeError) STRUCT_FOR_ID(PENDING) STRUCT_FOR_ID(Py_Repr) @@ -155,6 +169,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(__iter__) STRUCT_FOR_ID(__itruediv__) STRUCT_FOR_ID(__ixor__) + STRUCT_FOR_ID(__lazy_import__) + STRUCT_FOR_ID(__lazy_modules__) STRUCT_FOR_ID(__le__) STRUCT_FOR_ID(__len__) STRUCT_FOR_ID(__length_hint__) @@ -283,13 +299,16 @@ struct _Py_global_strings { STRUCT_FOR_ID(aclose) STRUCT_FOR_ID(add) STRUCT_FOR_ID(add_done_callback) + STRUCT_FOR_ID(adobe) STRUCT_FOR_ID(after_in_child) STRUCT_FOR_ID(after_in_parent) STRUCT_FOR_ID(alias) STRUCT_FOR_ID(align) STRUCT_FOR_ID(all) + STRUCT_FOR_ID(all_interpreters) STRUCT_FOR_ID(all_threads) STRUCT_FOR_ID(allow_code) + STRUCT_FOR_ID(alphabet) STRUCT_FOR_ID(any) STRUCT_FOR_ID(append) STRUCT_FOR_ID(arg) @@ -330,6 +349,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(c_parameter_type) STRUCT_FOR_ID(c_return) STRUCT_FOR_ID(c_stack) + STRUCT_FOR_ID(cache_frames) STRUCT_FOR_ID(cached_datetime_module) STRUCT_FOR_ID(cached_statements) STRUCT_FOR_ID(cadata) @@ -340,6 +360,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(callable) STRUCT_FOR_ID(callback) STRUCT_FOR_ID(cancel) + STRUCT_FOR_ID(canonical) STRUCT_FOR_ID(capath) STRUCT_FOR_ID(capitals) STRUCT_FOR_ID(category) @@ -373,9 +394,11 @@ struct _Py_global_strings { STRUCT_FOR_ID(co_varnames) STRUCT_FOR_ID(code) STRUCT_FOR_ID(col_offset) + STRUCT_FOR_ID(collector) STRUCT_FOR_ID(command) STRUCT_FOR_ID(comment_factory) STRUCT_FOR_ID(compile_mode) + STRUCT_FOR_ID(compression) STRUCT_FOR_ID(config) STRUCT_FOR_ID(consts) STRUCT_FOR_ID(context) @@ -424,6 +447,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(eager_start) STRUCT_FOR_ID(effective_ids) STRUCT_FOR_ID(element_factory) + STRUCT_FOR_ID(emptyerror) STRUCT_FOR_ID(encode) STRUCT_FOR_ID(encoding) STRUCT_FOR_ID(end) @@ -437,7 +461,9 @@ struct _Py_global_strings { STRUCT_FOR_ID(event) STRUCT_FOR_ID(eventmask) STRUCT_FOR_ID(exc) + STRUCT_FOR_ID(exc_tb) STRUCT_FOR_ID(exc_type) + STRUCT_FOR_ID(exc_val) STRUCT_FOR_ID(exc_value) STRUCT_FOR_ID(excepthook) STRUCT_FOR_ID(exception) @@ -449,6 +475,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(extra_tokens) STRUCT_FOR_ID(facility) STRUCT_FOR_ID(factory) + STRUCT_FOR_ID(fallback) STRUCT_FOR_ID(false) STRUCT_FOR_ID(family) STRUCT_FOR_ID(fanout) @@ -471,6 +498,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(flags) STRUCT_FOR_ID(flush) STRUCT_FOR_ID(fold) + STRUCT_FOR_ID(foldspaces) STRUCT_FOR_ID(follow_symlinks) STRUCT_FOR_ID(format) STRUCT_FOR_ID(format_spec) @@ -481,8 +509,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(fromtimestamp) STRUCT_FOR_ID(fromutc) STRUCT_FOR_ID(fset) + STRUCT_FOR_ID(fullerror) STRUCT_FOR_ID(func) STRUCT_FOR_ID(future) + STRUCT_FOR_ID(gc) STRUCT_FOR_ID(generation) STRUCT_FOR_ID(get) STRUCT_FOR_ID(get_debug) @@ -509,6 +539,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(ident) STRUCT_FOR_ID(identity_hint) STRUCT_FOR_ID(ignore) + STRUCT_FOR_ID(ignorechars) STRUCT_FOR_ID(imag) STRUCT_FOR_ID(implieslink) STRUCT_FOR_ID(importlib) @@ -587,12 +618,15 @@ struct _Py_global_strings { STRUCT_FOR_ID(loop) STRUCT_FOR_ID(manual_reset) STRUCT_FOR_ID(mapping) + STRUCT_FOR_ID(mask) STRUCT_FOR_ID(match) STRUCT_FOR_ID(max_length) + STRUCT_FOR_ID(max_threads) STRUCT_FOR_ID(maxdigits) STRUCT_FOR_ID(maxevents) STRUCT_FOR_ID(maxlen) STRUCT_FOR_ID(maxmem) + STRUCT_FOR_ID(maxsize) STRUCT_FOR_ID(maxsplit) STRUCT_FOR_ID(maxvalue) STRUCT_FOR_ID(memLevel) @@ -624,6 +658,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(name_from) STRUCT_FOR_ID(namespace_separator) STRUCT_FOR_ID(namespaces) + STRUCT_FOR_ID(native) STRUCT_FOR_ID(ndigits) STRUCT_FOR_ID(nested) STRUCT_FOR_ID(new_file_name) @@ -650,6 +685,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(only_keys) STRUCT_FOR_ID(oparg) STRUCT_FOR_ID(opcode) + STRUCT_FOR_ID(opcodes) STRUCT_FOR_ID(open) STRUCT_FOR_ID(opener) STRUCT_FOR_ID(operation) @@ -663,6 +699,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(outpath) STRUCT_FOR_ID(overlapped) STRUCT_FOR_ID(owner) + STRUCT_FOR_ID(pad) + STRUCT_FOR_ID(padded) STRUCT_FOR_ID(pages) STRUCT_FOR_ID(parameter) STRUCT_FOR_ID(parent) @@ -686,12 +724,16 @@ struct _Py_global_strings { STRUCT_FOR_ID(print_file_and_line) STRUCT_FOR_ID(priority) STRUCT_FOR_ID(progress) + STRUCT_FOR_ID(progress_callback) STRUCT_FOR_ID(progress_routine) STRUCT_FOR_ID(proto) STRUCT_FOR_ID(protocol) STRUCT_FOR_ID(ps1) STRUCT_FOR_ID(ps2) + STRUCT_FOR_ID(qid) + STRUCT_FOR_ID(qualname) STRUCT_FOR_ID(query) + STRUCT_FOR_ID(queuetype) STRUCT_FOR_ID(quotetabs) STRUCT_FOR_ID(raw) STRUCT_FOR_ID(read) @@ -703,6 +745,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(readline) STRUCT_FOR_ID(readonly) STRUCT_FOR_ID(real) + STRUCT_FOR_ID(recursive) STRUCT_FOR_ID(reducer_override) STRUCT_FOR_ID(registry) STRUCT_FOR_ID(rel_tol) @@ -722,6 +765,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(reversed) STRUCT_FOR_ID(rounding) STRUCT_FOR_ID(salt) + STRUCT_FOR_ID(sample_interval_us) STRUCT_FOR_ID(sched_priority) STRUCT_FOR_ID(scheduler) STRUCT_FOR_ID(script) @@ -752,6 +796,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(size) STRUCT_FOR_ID(sizehint) STRUCT_FOR_ID(skip_file_prefixes) + STRUCT_FOR_ID(skip_non_matching_threads) STRUCT_FOR_ID(sleep) STRUCT_FOR_ID(sock) STRUCT_FOR_ID(sort) @@ -760,9 +805,12 @@ struct _Py_global_strings { STRUCT_FOR_ID(spam) STRUCT_FOR_ID(src) STRUCT_FOR_ID(src_dir_fd) + STRUCT_FOR_ID(stack_frames) STRUCT_FOR_ID(stacklevel) STRUCT_FOR_ID(start) + STRUCT_FOR_ID(start_time_us) STRUCT_FOR_ID(statement) + STRUCT_FOR_ID(stats) STRUCT_FOR_ID(status) STRUCT_FOR_ID(stderr) STRUCT_FOR_ID(stdin) @@ -780,6 +828,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(symmetric_difference_update) STRUCT_FOR_ID(tabsize) STRUCT_FOR_ID(tag) + STRUCT_FOR_ID(take_bytes) STRUCT_FOR_ID(target) STRUCT_FOR_ID(target_is_directory) STRUCT_FOR_ID(task) @@ -800,6 +849,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(times) STRUCT_FOR_ID(timespec) STRUCT_FOR_ID(timestamp) + STRUCT_FOR_ID(timestamp_us) STRUCT_FOR_ID(timetuple) STRUCT_FOR_ID(timeunit) STRUCT_FOR_ID(top) @@ -817,6 +867,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(tzinfo) STRUCT_FOR_ID(tzname) STRUCT_FOR_ID(uid) + STRUCT_FOR_ID(unboundop) STRUCT_FOR_ID(unlink) STRUCT_FOR_ID(unraisablehook) STRUCT_FOR_ID(updates) @@ -837,6 +888,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(which) STRUCT_FOR_ID(who) STRUCT_FOR_ID(withdata) + STRUCT_FOR_ID(wrapcol) STRUCT_FOR_ID(writable) STRUCT_FOR_ID(write) STRUCT_FOR_ID(write_through) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 13fbff4eb65..32ed3a62b2b 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -17,7 +17,8 @@ extern int _PyImport_IsInitialized(PyInterpreterState *); // Export for 'pyexpat' shared extension PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); -extern int _PyImport_SetModuleString(const char *name, PyObject* module); +// Export for 'math' shared extension +PyAPI_FUNC(int) _PyImport_SetModuleString(const char *name, PyObject* module); extern void _PyImport_AcquireLock(PyInterpreterState *interp); extern void _PyImport_ReleaseLock(PyInterpreterState *interp); @@ -31,6 +32,18 @@ extern int _PyImport_FixupBuiltin( PyObject *modules ); +extern PyObject * _PyImport_ResolveName( + PyThreadState *tstate, PyObject *name, PyObject *globals, int level); +extern PyObject * _PyImport_GetAbsName( + PyThreadState *tstate, PyObject *name, PyObject *globals, int level); +// Symbol is exported for the JIT on Windows builds. +PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate( + PyThreadState *tstate, PyObject *lazy_import); +extern PyObject * _PyImport_LazyImportModuleLevelObject( + PyThreadState *tstate, PyObject *name, PyObject *builtins, + PyObject *globals, PyObject *locals, PyObject *fromlist, int level); + + #ifdef HAVE_DLOPEN # include // RTLD_NOW, RTLD_LAZY # if HAVE_DECL_RTLD_NOW @@ -68,11 +81,19 @@ extern void _PyImport_ClearModules(PyInterpreterState *interp); extern void _PyImport_ClearModulesByIndex(PyInterpreterState *interp); +extern PyObject * _PyImport_InitLazyModules( + PyInterpreterState *interp); +extern void _PyImport_ClearLazyModules(PyInterpreterState *interp); + extern int _PyImport_InitDefaultImportFunc(PyInterpreterState *interp); extern int _PyImport_IsDefaultImportFunc( PyInterpreterState *interp, PyObject *func); +extern int _PyImport_IsDefaultLazyImportFunc( + PyInterpreterState *interp, + PyObject *func); + extern PyObject * _PyImport_GetImportlibLoader( PyInterpreterState *interp, const char *loader_name); @@ -127,11 +148,18 @@ PyAPI_FUNC(int) _PyImport_ClearExtension(PyObject *name, PyObject *filename); // state of the module argument: // - If module is NULL or a PyModuleObject with md_gil == Py_MOD_GIL_NOT_USED, // call _PyEval_DisableGIL(). -// - Otherwise, call _PyEval_EnableGILPermanent(). If the GIL was not already -// enabled permanently, issue a warning referencing the module's name. +// - Otherwise, call _PyImport_EnableGILAndWarn // // This function may raise an exception. extern int _PyImport_CheckGILForModule(PyObject *module, PyObject *module_name); +// Assuming that the GIL is enabled from a call to +// _PyEval_EnableGILTransient(), call _PyEval_EnableGILPermanent(). +// If the GIL was not already enabled permanently, issue a warning referencing +// the module's name. +// Leave a message in verbose mode. +// +// This function may raise an exception. +extern int _PyImport_EnableGILAndWarn(PyThreadState *, PyObject *module_name); #endif #ifdef __cplusplus diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index 3ba9229cc21..f60c5510d20 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -14,6 +14,34 @@ extern "C" { extern const char *_PyImport_DynLoadFiletab[]; +#ifdef HAVE_DYNAMIC_LOADING +/* ./configure sets HAVE_DYNAMIC_LOADING if dynamic loading of modules is + supported on this platform. configure will then compile and link in one + of the dynload_*.c files, as appropriate. We will call a function in + those modules to get a function pointer to the module's init function. + + The function should return: + - The function pointer on success + - NULL with exception set if the library cannot be loaded + - NULL *without* an extension set if the library could be loaded but the + function cannot be found in it. +*/ +#ifdef MS_WINDOWS +#include +typedef FARPROC dl_funcptr; +extern dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, + const char *shortname, + PyObject *pathname, + FILE *fp); +#else +typedef void (*dl_funcptr)(void); +extern dl_funcptr _PyImport_FindSharedFuncptr(const char *prefix, + const char *shortname, + const char *pathname, FILE *fp); +#endif + +#endif /* HAVE_DYNAMIC_LOADING */ + typedef enum ext_module_kind { _Py_ext_module_kind_UNKNOWN = 0, @@ -28,6 +56,11 @@ typedef enum ext_module_origin { _Py_ext_module_origin_DYNAMIC = 3, } _Py_ext_module_origin; +struct hook_prefixes { + const char *const init_prefix; + const char *const export_prefix; +}; + /* Input for loading an extension module. */ struct _Py_ext_module_loader_info { PyObject *filename; @@ -40,7 +73,7 @@ struct _Py_ext_module_loader_info { * depending on if it's builtin or not. */ PyObject *path; _Py_ext_module_origin origin; - const char *hook_prefix; + const struct hook_prefixes *hook_prefixes; const char *newcontext; }; extern void _Py_ext_module_loader_info_clear( @@ -62,7 +95,9 @@ extern int _Py_ext_module_loader_info_init_from_spec( PyObject *spec); #endif -/* The result from running an extension module's init function. */ +/* The result from running an extension module's init function. + * Not used for modules defined via PyModExport (slots array). + */ struct _Py_ext_module_loader_result { PyModuleDef *def; PyObject *module; @@ -89,10 +124,11 @@ extern void _Py_ext_module_loader_result_apply_error( /* The module init function. */ typedef PyObject *(*PyModInitFunction)(void); +typedef PyModuleDef_Slot *(*PyModExportFunction)(void); #ifdef HAVE_DYNAMIC_LOADING -extern PyModInitFunction _PyImport_GetModInitFunc( +extern int _PyImport_GetModuleExportHooks( struct _Py_ext_module_loader_info *info, - FILE *fp); + FILE *fp, PyModInitFunction *modinit, PyModExportFunction *modexport); #endif extern int _PyImport_RunModInitFunc( PyModInitFunction p0, @@ -104,8 +140,6 @@ extern int _PyImport_RunModInitFunc( #define MAXSUFFIXSIZE 12 #ifdef MS_WINDOWS -#include -typedef FARPROC dl_funcptr; #ifdef Py_DEBUG # define PYD_DEBUG_SUFFIX "_d" @@ -128,8 +162,6 @@ typedef FARPROC dl_funcptr; #define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX "." PYD_SOABI ".pyd" #define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" -#else -typedef void (*dl_funcptr)(void); #endif diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 368dafb9063..183b2d45c5e 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -153,10 +153,8 @@ typedef enum { } _PyConfigInitEnum; typedef enum { - /* For now, this means the GIL is enabled. - - gh-116329: This will eventually change to "the GIL is disabled but can - be re-enabled by loading an incompatible extension module." */ + /* In free threaded builds, this means that the GIL is disabled at startup, + but may be enabled by loading an incompatible extension module. */ _PyConfig_GIL_DEFAULT = -1, /* The GIL has been forced off or on, and will not be affected by module loading. */ diff --git a/Include/internal/pycore_instruction_sequence.h b/Include/internal/pycore_instruction_sequence.h index b5c92773537..6257b771fd9 100644 --- a/Include/internal/pycore_instruction_sequence.h +++ b/Include/internal/pycore_instruction_sequence.h @@ -73,6 +73,7 @@ int _PyInstructionSequence_SetAnnotationsCode(_PyInstructionSequence *seq, _PyInstructionSequence *annotations); int _PyInstructionSequence_AddNested(_PyInstructionSequence *seq, _PyInstructionSequence *nested); void PyInstructionSequence_Fini(_PyInstructionSequence *seq); +_PyInstruction _PyInstructionSequence_GetInstruction(_PyInstructionSequence *seq, int pos); extern PyTypeObject _PyInstructionSequence_Type; #define _PyInstructionSequence_Check(v) Py_IS_TYPE((v), &_PyInstructionSequence_Type) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 7658adca719..56b55e93a01 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -33,54 +33,52 @@ int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events); int _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet events); int _PyMonitoring_GetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet *events); -extern int + +// these are exported only for other re-generated interpreters to call +PyAPI_FUNC(int) _Py_call_instrumentation(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); -extern int +PyAPI_FUNC(int) _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev); -extern int +PyAPI_FUNC(int) _Py_call_instrumentation_instruction( PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr); -_Py_CODEUNIT * +PyAPI_FUNC(_Py_CODEUNIT *) _Py_call_instrumentation_jump( _Py_CODEUNIT *instr, PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest); -extern int +PyAPI_FUNC(int) _Py_call_instrumentation_arg(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg); -extern int +PyAPI_FUNC(int) _Py_call_instrumentation_2args(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); -extern void +PyAPI_FUNC(void) _Py_call_instrumentation_exc2(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); -extern int -_Py_Instrumentation_GetLine(PyCodeObject *code, int index); - -extern PyObject _PyInstrumentation_MISSING; -extern PyObject _PyInstrumentation_DISABLE; +PyAPI_DATA(PyObject) _PyInstrumentation_MISSING; +PyAPI_DATA(PyObject) _PyInstrumentation_DISABLE; /* Total tool ids available */ #define PY_MONITORING_TOOL_IDS 8 -/* Count of all local monitoring events */ -#define _PY_MONITORING_LOCAL_EVENTS 11 -/* Count of all "real" monitoring events (not derived from other events) */ +/* Count of all "real" monitoring events (not derived from other events). + * "Other" events can now be turned on/disabled per code object. */ #define _PY_MONITORING_UNGROUPED_EVENTS 16 /* Count of all monitoring events */ #define _PY_MONITORING_EVENTS 19 /* Tables of which tools are active for each monitored event. */ typedef struct _Py_LocalMonitors { - uint8_t tools[_PY_MONITORING_LOCAL_EVENTS]; + uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS]; } _Py_LocalMonitors; typedef struct _Py_GlobalMonitors { @@ -120,6 +118,17 @@ typedef struct _PyCoMonitoringData { uint8_t *per_instruction_tools; } _PyCoMonitoringData; +extern int +_Py_Instrumentation_GetLine(PyCodeObject *code, _PyCoLineInstrumentationData *line_data, int index); + +static inline uint8_t +_PyCode_GetOriginalOpcode(_PyCoLineInstrumentationData *line_data, int index) +{ + return line_data->data[index*line_data->bytes_per_entry]; +} + +// Exported for external JIT support +PyAPI_FUNC(uint8_t) _PyCode_Deinstrument(uint8_t opcode); #ifdef __cplusplus } diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index fa9568ab4d0..86f018e3286 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -14,7 +14,7 @@ extern "C" { #include "pycore_structs.h" // PyHamtObject #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_typedefs.h" // _PyRuntimeState - +#include "pycore_uop.h" // _PyBloomFilter #define CODE_MAX_WATCHERS 8 #define CONTEXT_MAX_WATCHERS 8 @@ -69,7 +69,7 @@ struct code_arena_st; struct trampoline_api_st { void* (*init_state)(void); void (*write_state)(void* state, const void *code_addr, - unsigned int code_size, PyCodeObject* code); + size_t code_size, PyCodeObject* code); int (*free_state)(void* state); void *state; Py_ssize_t code_padding; @@ -88,7 +88,9 @@ struct _ceval_runtime_state { struct trampoline_api_st trampoline_api; FILE *map_file; Py_ssize_t persist_after_fork; - _PyFrameEvalFunction prev_eval_frame; + _PyFrameEvalFunction prev_eval_frame; + Py_ssize_t trampoline_refcount; + int code_watcher_id; #else int _not_used; #endif @@ -175,62 +177,73 @@ struct gc_generation { generations */ }; -struct gc_collection_stats { - /* number of collected objects */ - Py_ssize_t collected; - /* total number of uncollectable objects (put into gc.garbage) */ - Py_ssize_t uncollectable; -}; - /* Running stats per generation */ struct gc_generation_stats { + PyTime_t ts_start; + PyTime_t ts_stop; /* total number of collections */ Py_ssize_t collections; /* total number of collected objects */ Py_ssize_t collected; /* total number of uncollectable objects (put into gc.garbage) */ Py_ssize_t uncollectable; + // Total number of objects considered for collection and traversed: + Py_ssize_t candidates; + // Total duration of the collection in seconds: + double duration; }; -enum _GCPhase { - GC_PHASE_MARK = 0, - GC_PHASE_COLLECT = 1 +#ifdef Py_GIL_DISABLED +#define GC_YOUNG_STATS_SIZE 1 +#define GC_OLD_STATS_SIZE 1 +#else +#define GC_YOUNG_STATS_SIZE 11 +#define GC_OLD_STATS_SIZE 3 +#endif +struct gc_young_stats_buffer { + struct gc_generation_stats items[GC_YOUNG_STATS_SIZE]; + int8_t index; +}; + +struct gc_old_stats_buffer { + struct gc_generation_stats items[GC_OLD_STATS_SIZE]; + int8_t index; }; /* If we change this, we need to change the default value in the - signature of gc.collect. */ + signature of gc.collect and change the size of PyStats.gc_stats */ #define NUM_GENERATIONS 3 -struct _gc_runtime_state { - /* List of objects that still need to be cleaned up, singly linked - * via their gc headers' gc_prev pointers. */ - PyObject *trash_delete_later; - /* Current call-stack depth of tp_dealloc calls. */ - int trash_delete_nesting; +struct gc_stats { + struct gc_young_stats_buffer young; + struct gc_old_stats_buffer old[2]; +}; +struct _gc_runtime_state { /* Is automatic collection enabled? */ int enabled; int debug; /* linked lists of container objects */ +#ifndef Py_GIL_DISABLED + struct gc_generation generations[NUM_GENERATIONS]; + PyGC_Head *generation0; +#else struct gc_generation young; struct gc_generation old[2]; +#endif /* a permanent generation which won't be collected */ struct gc_generation permanent_generation; - struct gc_generation_stats generation_stats[NUM_GENERATIONS]; + struct gc_stats *generation_stats; /* true if we are currently running the collector */ int collecting; + // The frame that started the current collection. It might be NULL even when + // collecting (if no Python frame is running): + _PyInterpreterFrame *frame; /* list of uncollectable objects */ PyObject *garbage; /* a list of callbacks to be invoked when collection is performed */ PyObject *callbacks; - Py_ssize_t heap_size; - Py_ssize_t work_to_do; - /* Which of the old spaces is the visited space */ - int visited_space; - int phase; - -#ifdef Py_GIL_DISABLED /* This is the number of objects that survived the last full collection. It approximates the number of long lived objects tracked by the GC. @@ -243,6 +256,7 @@ struct _gc_runtime_state { the first time. */ Py_ssize_t long_lived_pending; +#ifdef Py_GIL_DISABLED /* True if gc.freeze() has been used. */ int freeze_active; @@ -258,6 +272,22 @@ struct _gc_runtime_state { #endif }; +#ifndef Py_GIL_DISABLED +#define GC_GENERATION_INIT \ + .generations = { \ + { .threshold = 2000, }, \ + { .threshold = 10, }, \ + { .threshold = 10, }, \ + }, +#else +#define GC_GENERATION_INIT \ + .young = { .threshold = 2000, }, \ + .old = { \ + { .threshold = 10, }, \ + { .threshold = 10, }, \ + }, +#endif + #include "pycore_gil.h" // struct _gil_runtime_state /**** Import ********/ @@ -318,6 +348,14 @@ struct _import_state { int dlopenflags; #endif PyObject *import_func; + PyObject *lazy_import_func; + int lazy_imports_mode; + PyObject *lazy_imports_filter; + PyObject *lazy_importing_modules; + PyObject *lazy_modules; +#ifdef Py_GIL_DISABLED + PyMutex lazy_mutex; +#endif /* The global import lock. */ _PyRecursiveMutex lock; /* diagnostic info in PyImport_ImportModuleLevelObject() */ @@ -392,6 +430,28 @@ typedef struct _rare_events { uint8_t func_modification; } _rare_events; +// Optimization configuration for the interpreter. +// This groups all thresholds and optimization flags for both JIT and interpreter. +typedef struct _PyOptimizationConfig { + // Interpreter optimization thresholds + uint16_t jump_backward_initial_value; + uint16_t jump_backward_initial_backoff; + + uint16_t resume_initial_value; + uint16_t resume_initial_backoff; + + // JIT optimization thresholds + uint16_t side_exit_initial_value; + uint16_t side_exit_initial_backoff; + + // Trace fitness thresholds + uint16_t fitness_initial; + + // Optimization flags + bool specialization_enabled; + bool uops_optimize_enabled; +} _PyOptimizationConfig; + struct Bigint { struct Bigint *next; @@ -465,8 +525,13 @@ struct _py_func_state { /****** type state *********/ /* For now we hard-code this to a value for which we are confident - all the static builtin types will fit (for all builds). */ -#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES 200 + all the static builtin types will fit (for all builds). + If you add a new static type to the standard library, you may have to + update one of these numbers. + */ +#define _Py_NUM_MANAGED_PREINITIALIZED_TYPES 120 +#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES \ + (_Py_NUM_MANAGED_PREINITIALIZED_TYPES + 83) #define _Py_MAX_MANAGED_STATIC_EXT_TYPES 10 #define _Py_MAX_MANAGED_STATIC_TYPES \ (_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES + _Py_MAX_MANAGED_STATIC_EXT_TYPES) @@ -677,11 +742,6 @@ struct _Py_interp_cached_objects { /* object.__reduce__ */ PyObject *objreduce; -#ifndef Py_GIL_DISABLED - /* resolve_slotdups() */ - PyObject *type_slots_pname; - pytype_slotdef *type_slots_ptrs[MAX_EQUIV]; -#endif /* TypeVar and related types */ PyTypeObject *generic_type; @@ -779,12 +839,6 @@ struct _is { * and should be placed at the beginning. */ struct _ceval_state ceval; - /* This structure is carefully allocated so that it's correctly aligned - * to avoid undefined behaviors during LOAD and STORE. The '_malloced' - * field stores the allocated pointer address that will later be freed. - */ - void *_malloced; - PyInterpreterState *next; int64_t id; @@ -876,6 +930,7 @@ struct _is { PyObject *builtins_copy; // Initialized to _PyEval_EvalFrameDefault(). _PyFrameEvalFunction eval_frame; + int eval_frame_allow_specialization; PyFunction_WatchCallback func_watchers[FUNC_MAX_WATCHERS]; // One bit is set for each non-NULL entry in func_watchers @@ -949,11 +1004,18 @@ struct _is { struct callable_cache callable_cache; PyObject *common_consts[NUM_COMMON_CONSTANTS]; bool jit; - struct _PyExecutorObject *executor_list_head; + bool compiling; + + // Optimization configuration (thresholds and flags for JIT and interpreter) + _PyOptimizationConfig opt_config; + _PyBloomFilter *executor_blooms; // Contiguous bloom filter array + struct _PyExecutorObject **executor_ptrs; // Corresponding executor pointer array + size_t executor_count; // Number of valid executors + size_t executor_capacity; // Array capacity struct _PyExecutorObject *executor_deletion_list_head; struct _PyExecutorObject *cold_executor; - int executor_deletion_list_remaining_capacity; - size_t trace_run_counter; + struct _PyExecutorObject *cold_dynamic_executor; + size_t executor_creation_counter; _rare_events rare_events; PyDict_WatchCallback builtins_dict_watcher; @@ -977,6 +1039,18 @@ struct _is { # ifdef Py_STACKREF_CLOSE_DEBUG _Py_hashtable_t *closed_stackrefs_table; # endif +#endif + +#ifdef Py_STATS + // true if recording of pystats is on, this is used when new threads + // are created to decide if recording should be on for them + int pystats_enabled; + // allocated when (and if) stats are first enabled + PyStats *pystats_struct; +#ifdef Py_GIL_DISABLED + // held when pystats related interpreter state is being updated + PyMutex pystats_mutex; +#endif #endif /* the initial PyInterpreterState.threads.head */ diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index 2ee3696317c..28370ababc4 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -24,6 +24,36 @@ static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { return (PyCodeObject *)executable; } +// Similar to _PyFrame_GetCode(), but return NULL if the frame is invalid or +// freed. Used by dump_frame() in Python/traceback.c. The function uses +// heuristics to detect freed memory, it's not 100% reliable. +static inline PyCodeObject* _Py_NO_SANITIZE_THREAD +_PyFrame_SafeGetCode(_PyInterpreterFrame *f) +{ + // globals and builtins may be NULL on a legit frame, but it's unlikely. + // It's more likely that it's a sign of an invalid frame. + if (f->f_globals == NULL || f->f_builtins == NULL) { + return NULL; + } + + if (PyStackRef_IsNull(f->f_executable)) { + return NULL; + } + void *ptr; + memcpy(&ptr, &f->f_executable, sizeof(f->f_executable)); + if (_PyMem_IsPtrFreed(ptr)) { + return NULL; + } + PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable); + if (_PyObject_IsFreed(executable)) { + return NULL; + } + if (!PyCode_Check(executable)) { + return NULL; + } + return (PyCodeObject *)executable; +} + static inline _Py_CODEUNIT * _PyFrame_GetBytecode(_PyInterpreterFrame *f) { @@ -37,6 +67,31 @@ _PyFrame_GetBytecode(_PyInterpreterFrame *f) #endif } +// Similar to PyUnstable_InterpreterFrame_GetLasti(), but return NULL if the +// frame is invalid or freed. Used by dump_frame() in Python/traceback.c. The +// function uses heuristics to detect freed memory, it's not 100% reliable. +static inline int _Py_NO_SANITIZE_THREAD +_PyFrame_SafeGetLasti(struct _PyInterpreterFrame *f) +{ + // Code based on _PyFrame_GetBytecode() but replace _PyFrame_GetCode() + // with _PyFrame_SafeGetCode(). + PyCodeObject *co = _PyFrame_SafeGetCode(f); + if (co == NULL) { + return -1; + } + + _Py_CODEUNIT *bytecode; +#ifdef Py_GIL_DISABLED + _PyCodeArray *tlbc = _PyCode_GetTLBCArray(co); + assert(f->tlbc_index >= 0 && f->tlbc_index < tlbc->size); + bytecode = (_Py_CODEUNIT *)tlbc->entries[f->tlbc_index]; +#else + bytecode = _PyCode_CODE(co); +#endif + + return (int)(f->instr_ptr - bytecode) * sizeof(_Py_CODEUNIT); +} + static inline PyFunctionObject *_PyFrame_GetFunction(_PyInterpreterFrame *f) { PyObject *func = PyStackRef_AsPyObjectBorrow(f->f_funcobj); assert(PyFunction_Check(func)); @@ -47,10 +102,10 @@ static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) { return (f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); } -static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f) { +static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f, int depth) { assert(f->stackpointer > _PyFrame_Stackbase(f)); - assert(!PyStackRef_IsNull(f->stackpointer[-1])); - return f->stackpointer[-1]; + assert(!PyStackRef_IsNull(f->stackpointer[-depth])); + return f->stackpointer[-depth]; } static inline _PyStackRef _PyFrame_StackPop(_PyInterpreterFrame *f) { @@ -94,6 +149,11 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * int stacktop = (int)(src->stackpointer - src->localsplus); assert(stacktop >= 0); dest->stackpointer = dest->localsplus + stacktop; + // visited is GC bookkeeping for the current stack walk, not frame state. + dest->visited = 0; +#ifdef Py_DEBUG + dest->lltrace = src->lltrace; +#endif for (int i = 0; i < stacktop; i++) { dest->localsplus[i] = PyStackRef_MakeHeapSafe(src->localsplus[i]); } @@ -224,7 +284,7 @@ _PyThreadState_GetFrame(PyThreadState *tstate) /* For use by _PyFrame_GetFrameObject Do not call directly. */ -PyFrameObject * +PyAPI_FUNC(PyFrameObject *) _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame); /* Gets the PyFrameObject for this frame, lazily @@ -242,7 +302,8 @@ _PyFrame_GetFrameObject(_PyInterpreterFrame *frame) return _PyFrame_MakeAndSetFrameObject(frame); } -void +// Exported for external JIT support +PyAPI_FUNC(void) _PyFrame_ClearLocals(_PyInterpreterFrame *frame); /* Clears all references in the frame. @@ -253,8 +314,10 @@ _PyFrame_ClearLocals(_PyInterpreterFrame *frame); * in the frame. * take should be set to 1 for heap allocated * frames like the ones in generators and coroutines. + * + * Exported for external JIT support */ -void + PyAPI_FUNC(void) _PyFrame_ClearExceptCode(_PyInterpreterFrame * frame); int @@ -278,7 +341,8 @@ _PyThreadState_HasStackSpace(PyThreadState *tstate, int size) size < tstate->datastack_limit - tstate->datastack_top; } -extern _PyInterpreterFrame * +// Exported for external JIT support +PyAPI_FUNC(_PyInterpreterFrame *) _PyThreadState_PushFrame(PyThreadState *tstate, size_t size); PyAPI_FUNC(void) _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame *frame); @@ -340,6 +404,10 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func, size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous); +PyAPI_FUNC(_PyInterpreterFrame *) +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func, + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 835b8e58194..38510685f40 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -24,7 +24,6 @@ enum _frameowner { FRAME_OWNED_BY_GENERATOR = 1, FRAME_OWNED_BY_FRAME_OBJECT = 2, FRAME_OWNED_BY_INTERPRETER = 3, - FRAME_OWNED_BY_CSTACK = 4, }; struct _PyInterpreterFrame { diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h index 8a88cbf607b..2f97cc26eaf 100644 --- a/Include/internal/pycore_jit.h +++ b/Include/internal/pycore_jit.h @@ -13,12 +13,24 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +/* To be able to reason about code layout and branches, keep code size below 1 MB */ +#define PY_MAX_JIT_CODE_SIZE ((1 << 20)-1) + #ifdef _Py_JIT -typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); +typedef _Py_CODEUNIT *(*jit_func)( + _PyExecutorObject *executor, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, + _PyStackRef _tos_cache0, _PyStackRef _tos_cache1, _PyStackRef _tos_cache2 +); + +_Py_CODEUNIT *_PyJIT_Entry( + _PyExecutorObject *executor, _PyInterpreterFrame *frame, + _PyStackRef *stack_pointer, PyThreadState *tstate +); int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length); void _PyJIT_Free(_PyExecutorObject *executor); +PyAPI_FUNC(int) _PyJIT_AddressInJitCode(PyInterpreterState *interp, uintptr_t addr); #endif // _Py_JIT diff --git a/Include/internal/pycore_jit_unwind.h b/Include/internal/pycore_jit_unwind.h new file mode 100644 index 00000000000..2d325ad9562 --- /dev/null +++ b/Include/internal/pycore_jit_unwind.h @@ -0,0 +1,68 @@ +#ifndef Py_INTERNAL_JIT_UNWIND_H +#define Py_INTERNAL_JIT_UNWIND_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include +#include + +#if defined(_Py_JIT) && defined(__linux__) && defined(__ELF__) +# define PY_HAVE_JIT_GDB_UNWIND +#endif + +#if defined(PY_HAVE_PERF_TRAMPOLINE) || defined(PY_HAVE_JIT_GDB_UNWIND) + +#if defined(PY_HAVE_JIT_GDB_UNWIND) +extern PyMutex _Py_jit_debug_mutex; +#endif + +/* DWARF exception-handling pointer encodings shared by JIT unwind users. */ +enum { + DWRF_EH_PE_absptr = 0x00, + DWRF_EH_PE_omit = 0xff, + + /* Data type encodings */ + DWRF_EH_PE_uleb128 = 0x01, + DWRF_EH_PE_udata2 = 0x02, + DWRF_EH_PE_udata4 = 0x03, + DWRF_EH_PE_udata8 = 0x04, + DWRF_EH_PE_sleb128 = 0x09, + DWRF_EH_PE_sdata2 = 0x0a, + DWRF_EH_PE_sdata4 = 0x0b, + DWRF_EH_PE_sdata8 = 0x0c, + DWRF_EH_PE_signed = 0x08, + + /* Reference type encodings */ + DWRF_EH_PE_pcrel = 0x10, + DWRF_EH_PE_textrel = 0x20, + DWRF_EH_PE_datarel = 0x30, + DWRF_EH_PE_funcrel = 0x40, + DWRF_EH_PE_aligned = 0x50, + DWRF_EH_PE_indirect = 0x80 +}; + +/* Return the size of the generated .eh_frame data for the given encoding. */ +size_t _PyJitUnwind_EhFrameSize(int absolute_addr); + +/* + * Build DWARF .eh_frame data for JIT code; returns size written or 0 on error. + * absolute_addr selects the FDE address encoding: + * - 0: PC-relative offsets (perf jitdump synthesized DSO). + * - nonzero: absolute addresses (GDB JIT in-memory ELF). + */ +size_t _PyJitUnwind_BuildEhFrame(uint8_t *buffer, size_t buffer_size, + const void *code_addr, size_t code_size, + int absolute_addr); + +void *_PyJitUnwind_GdbRegisterCode(const void *code_addr, + size_t code_size, + const char *entry, + const char *filename); + +void _PyJitUnwind_GdbUnregisterCode(void *handle); + +#endif // defined(PY_HAVE_PERF_TRAMPOLINE) || defined(PY_HAVE_JIT_GDB_UNWIND) + +#endif // Py_INTERNAL_JIT_UNWIND_H diff --git a/Include/internal/pycore_lazyimportobject.h b/Include/internal/pycore_lazyimportobject.h new file mode 100644 index 00000000000..b81e4211b08 --- /dev/null +++ b/Include/internal/pycore_lazyimportobject.h @@ -0,0 +1,35 @@ +// Lazy object interface. + +#ifndef Py_INTERNAL_LAZYIMPORTOBJECT_H +#define Py_INTERNAL_LAZYIMPORTOBJECT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +PyAPI_DATA(PyTypeObject) PyLazyImport_Type; +#define PyLazyImport_CheckExact(op) Py_IS_TYPE((op), &PyLazyImport_Type) + +typedef struct { + PyObject_HEAD + PyObject *lz_builtins; + PyObject *lz_from; + PyObject *lz_attr; + // Frame information for the original import location. + PyCodeObject *lz_code; // Code object where the lazy import was created. + int lz_instr_offset; // Instruction offset where the lazy import was created. +} PyLazyImportObject; + + +PyAPI_FUNC(PyObject *) _PyLazyImport_GetName(PyObject *lazy_import); +PyAPI_FUNC(PyObject *) _PyLazyImport_New( + struct _PyInterpreterFrame *frame, PyObject *import_func, PyObject *from, PyObject *attr); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_LAZYIMPORTOBJECT_H diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index ffbcebdb7df..df0d00f7525 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -14,15 +14,17 @@ extern "C" { PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *); PyAPI_FUNC(PyObject) *_PyList_SliceSubscript(PyObject*, PyObject*); +PyAPI_FUNC(PyObject *) _PyList_BinarySlice(PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyList_Concat(PyObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); // _PyList_GetItemRef should be used only when the object is known as a list // because it doesn't raise TypeError when the object is not a list, whereas PyList_GetItemRef does. -extern PyObject* _PyList_GetItemRef(PyListObject *, Py_ssize_t i); +PyAPI_FUNC(PyObject *) _PyList_GetItemRef(PyListObject *, Py_ssize_t i); #ifdef Py_GIL_DISABLED // Returns -1 in case of races with other threads. -extern int _PyList_GetItemRefNoLock(PyListObject *, Py_ssize_t, _PyStackRef *); +PyAPI_FUNC(int) _PyList_GetItemRefNoLock(PyListObject *, Py_ssize_t, _PyStackRef *); #endif #define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item) diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index c4e007e744c..e31d8b4e5c6 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -70,6 +70,9 @@ PyMutex_LockFlags(PyMutex *m, _PyLockFlags flags) // error messages) otherwise returns 0. extern int _PyMutex_TryUnlock(PyMutex *m); +// Yield the processor to other threads (e.g., sched_yield). +extern void _Py_yield(void); + // PyEvent is a one-time event notification typedef struct { diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 3c213783cd4..fb5622c99f7 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -64,7 +64,8 @@ PyAPI_FUNC(void) _PyLong_ExactDealloc(PyObject *self); # error "_PY_NSMALLPOSINTS must be greater than or equal to 257" #endif -#define _PY_IS_SMALL_INT(val) ((val) >= 0 && (val) < 256 && (val) < _PY_NSMALLPOSINTS) +#define _PY_IS_SMALL_INT(val) \ + (-_PY_NSMALLNEGINTS <= (val) && (val) < _PY_NSMALLPOSINTS) // Return a reference to the immortal zero singleton. // The function cannot return NULL. @@ -135,7 +136,7 @@ extern int _PyLong_FormatWriter( int alternate); extern char* _PyLong_FormatBytesWriter( - _PyBytesWriter *writer, + PyBytesWriter *writer, char *str, PyObject *obj, int base, @@ -231,6 +232,21 @@ _PyLong_IsPositive(const PyLongObject *op) return (op->long_value.lv_tag & SIGN_MASK) == 0; } +/* Return true if the argument is a small int */ +static inline bool +_PyLong_IsSmallInt(const PyLongObject *op) +{ + assert(PyLong_Check(op)); + bool is_small_int = (op->long_value.lv_tag & IMMORTALITY_BIT_MASK) != 0; + if (is_small_int) { + assert(PyLong_CheckExact(op)); + assert(_Py_IsImmortal(op)); + assert((_PyLong_IsCompact(op) + && _PY_IS_SMALL_INT(_PyLong_CompactValue(op)))); + } + return is_small_int; +} + static inline Py_ssize_t _PyLong_DigitCount(const PyLongObject *op) { @@ -270,6 +286,14 @@ _PyLong_SameSign(const PyLongObject *a, const PyLongObject *b) return (a->long_value.lv_tag & SIGN_MASK) == (b->long_value.lv_tag & SIGN_MASK); } +/* Initialize the tag of a freshly-allocated int. */ +static inline void +_PyLong_InitTag(PyLongObject *op) +{ + assert(PyLong_Check(op)); + op->long_value.lv_tag = SIGN_ZERO; /* non-immortal zero */ +} + #define TAG_FROM_SIGN_AND_SIZE(sign, size) \ ((uintptr_t)(1 - (sign)) | ((uintptr_t)(size) << NON_SIZE_BITS)) @@ -279,6 +303,7 @@ _PyLong_SetSignAndDigitCount(PyLongObject *op, int sign, Py_ssize_t size) assert(size >= 0); assert(-1 <= sign && sign <= 1); assert(sign != 0 || size == 0); + assert(!_PyLong_IsSmallInt(op)); op->long_value.lv_tag = TAG_FROM_SIGN_AND_SIZE(sign, size); } @@ -286,13 +311,16 @@ static inline void _PyLong_SetDigitCount(PyLongObject *op, Py_ssize_t size) { assert(size >= 0); + assert(!_PyLong_IsSmallInt(op)); op->long_value.lv_tag = (((size_t)size) << NON_SIZE_BITS) | (op->long_value.lv_tag & SIGN_MASK); } #define NON_SIZE_MASK ~(uintptr_t)((1 << NON_SIZE_BITS) - 1) static inline void -_PyLong_FlipSign(PyLongObject *op) { +_PyLong_FlipSign(PyLongObject *op) +{ + assert(!_PyLong_IsSmallInt(op)); unsigned int flipped_sign = 2 - (op->long_value.lv_tag & SIGN_MASK); op->long_value.lv_tag &= NON_SIZE_MASK; op->long_value.lv_tag |= flipped_sign; diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 81bfd162c7e..fd918e13f2d 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -279,11 +279,23 @@ Known values: Python 3.14b1 3624 (Don't optimize LOAD_FAST when local is killed by DELETE_FAST) Python 3.14b3 3625 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) Python 3.14rc2 3626 (Fix missing exception handlers in logical expression) + Python 3.14rc3 3627 (Fix miscompilation of some module-level annotations) Python 3.15a0 3650 (Initial version) Python 3.15a1 3651 (Simplify LOAD_CONST) Python 3.15a1 3652 (Virtual iterators) Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) Python 3.15a1 3654 (Fix missing exception handlers in logical expression) + Python 3.15a1 3655 (Fix miscompilation of some module-level annotations) + Python 3.15a1 3656 (Add TRACE_RECORD instruction, for platforms with switch based interpreter) + Python 3.15a4 3657 (Add BINARY_OP_SUBSCR_USTR_INT) + Python 3.15a4 3658 (Optimize bytecode for list/set called on genexp) + Python 3.15a4 3659 (Add CALL_FUNCTION_EX specialization) + Python 3.15a4 3660 (Change generator preamble code) + Python 3.15a4 3661 (Lazy imports IMPORT_NAME opcode changes) + Python 3.15a8 3662 (Add counter to RESUME) + Python 3.15a8 3663 (Merge GET_ITER and GET_YIELD_FROM_ITER. Modify SEND to make it a bit more like FOR_ITER) + Python 3.15a8 3664 (Fix __qualname__ for __annotate__ functions) + Python 3.15a8 3665 (Add FOR_ITER_VIRTUAL and GET_ITER specializations) Python 3.16 will start with 3700 @@ -297,7 +309,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3654 +#define PYC_MAGIC_NUMBER 3665 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_mmap.h b/Include/internal/pycore_mmap.h new file mode 100644 index 00000000000..897816db010 --- /dev/null +++ b/Include/internal/pycore_mmap.h @@ -0,0 +1,47 @@ +#ifndef Py_INTERNAL_MMAP_H +#define Py_INTERNAL_MMAP_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_pystate.h" + +#if defined(HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) +# include +# include +#endif + +#if defined(HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) +static inline int +_PyAnnotateMemoryMap(void *addr, size_t size, const char *name) +{ +#ifndef Py_DEBUG + if (!_Py_GetConfig()->dev_mode) { + return 0; + } +#endif + // The name length cannot exceed 80 (including the '\0'). + assert(strlen(name) < 80); + int res = prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, (unsigned long)addr, size, name); + if (res < 0) { + return -1; + } + return 0; +} +#else +static inline int +_PyAnnotateMemoryMap(void *Py_UNUSED(addr), size_t Py_UNUSED(size), const char *Py_UNUSED(name)) +{ + return 0; +} +#endif + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_MMAP_H diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index b170d7bce70..5bcfd17cec4 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -1,5 +1,8 @@ #ifndef Py_INTERNAL_MODULEOBJECT_H #define Py_INTERNAL_MODULEOBJECT_H + +#include + #ifdef __cplusplus extern "C" { #endif @@ -16,32 +19,53 @@ extern int _PyModule_IsPossiblyShadowing(PyObject *); extern int _PyModule_IsExtension(PyObject *obj); +extern int _PyModule_InitModuleDictWatcher(PyInterpreterState *interp); + +typedef int (*_Py_modexecfunc)(PyObject *); + typedef struct { PyObject_HEAD PyObject *md_dict; - PyModuleDef *md_def; void *md_state; PyObject *md_weaklist; // for logging purposes after md_dict is cleared PyObject *md_name; + bool md_token_is_def; /* if true, `md_token` is the PyModuleDef */ #ifdef Py_GIL_DISABLED - void *md_gil; + bool md_requires_gil; #endif + Py_ssize_t md_state_size; + traverseproc md_state_traverse; + inquiry md_state_clear; + freefunc md_state_free; + void *md_token; + _Py_modexecfunc md_exec; /* only set if md_token_is_def is true */ } PyModuleObject; -static inline PyModuleDef* _PyModule_GetDef(PyObject *mod) { - assert(PyModule_Check(mod)); - return ((PyModuleObject *)mod)->md_def; +#define _PyModule_CAST(op) \ + (assert(PyModule_Check(op)), _Py_CAST(PyModuleObject*, (op))) + +static inline PyModuleDef *_PyModule_GetDefOrNull(PyObject *arg) { + PyModuleObject *mod = _PyModule_CAST(arg); + if (mod->md_token_is_def) { + return (PyModuleDef *)mod->md_token; + } + return NULL; } +// Get md_token. Used in _DuringGC functions; must have no side effects. +static inline PyModuleDef *_PyModule_GetToken(PyObject *arg) { + PyModuleObject *mod = _PyModule_CAST(arg); + return (PyModuleDef *)mod->md_token; +} + +// Get md_state. Used in _DuringGC functions; must have no side effects. static inline void* _PyModule_GetState(PyObject* mod) { - assert(PyModule_Check(mod)); - return ((PyModuleObject *)mod)->md_state; + return _PyModule_CAST(mod)->md_state; } static inline PyObject* _PyModule_GetDict(PyObject *mod) { - assert(PyModule_Check(mod)); - PyObject *dict = ((PyModuleObject *)mod) -> md_dict; + PyObject *dict = _PyModule_CAST(mod)->md_dict; // _PyModule_GetDict(mod) must not be used after calling module_clear(mod) assert(dict != NULL); return dict; // borrowed reference diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 40f8ca68c00..c2c508c1a71 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -144,7 +144,7 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) new_refcnt = _Py_IMMORTAL_INITIAL_REFCNT; } # if SIZEOF_VOID_P > 4 - op->ob_refcnt = (PY_UINT32_T)new_refcnt; + op->ob_refcnt = (uint32_t)new_refcnt; # else op->ob_refcnt = new_refcnt; # endif @@ -252,25 +252,6 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) } } -static inline void -_Py_DECREF_NO_DEALLOC(PyObject *op) -{ - if (_Py_IsImmortal(op)) { - _Py_DECREF_IMMORTAL_STAT_INC(); - return; - } - _Py_DECREF_STAT_INC(); -#ifdef Py_REF_DEBUG - _Py_DEC_REFTOTAL(PyInterpreterState_Get()); -#endif - op->ob_refcnt--; -#ifdef Py_DEBUG - if (op->ob_refcnt <= 0) { - _Py_FatalRefcountError("Expected a positive remaining refcount"); - } -#endif -} - #else // TODO: implement Py_DECREF specializations for Py_GIL_DISABLED build static inline void @@ -279,12 +260,6 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) Py_DECREF(op); } -static inline void -_Py_DECREF_NO_DEALLOC(PyObject *op) -{ - Py_DECREF(op); -} - static inline int _Py_REF_IS_MERGED(Py_ssize_t ob_ref_shared) { @@ -496,6 +471,9 @@ static inline void Py_DECREF_MORTAL_SPECIALIZED(PyObject *op, destructor destruc #define Py_DECREF_MORTAL_SPECIALIZED(op, destruct) Py_DECREF_MORTAL_SPECIALIZED(_PyObject_CAST(op), destruct) #endif +#else // Py_GIL_DISABLED +# define Py_DECREF_MORTAL(op) Py_DECREF(op) +# define Py_DECREF_MORTAL_SPECIALIZED(op, destruct) Py_DECREF(op) #endif /* Inline functions trading binary compatibility for speed: @@ -863,8 +841,7 @@ static inline Py_hash_t _PyObject_HashFast(PyObject *op) { if (PyUnicode_CheckExact(op)) { - Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED( - _PyASCIIObject_CAST(op)->hash); + Py_hash_t hash = PyUnstable_Unicode_GET_CACHED_HASH(op); if (hash != -1) { return hash; } @@ -902,14 +879,16 @@ PyAPI_FUNC(PyObject *) _PyType_NewManagedObject(PyTypeObject *type); extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int); -extern int _PyObject_SetAttributeErrorContext(PyObject *v, PyObject* name); +// Exported for external JIT support +PyAPI_FUNC(int) _PyObject_SetAttributeErrorContext(PyObject *v, PyObject* name); void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyObject *name, PyObject *value); extern bool _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr); -extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, unsigned int *); // Internal API to look for a name through the MRO. @@ -918,9 +897,13 @@ extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, extern unsigned int _PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef *out); -PyAPI_FUNC(int) _PyObject_GetMethodStackRef(PyThreadState *ts, PyObject *obj, +extern int _PyObject_GetMethodStackRef(PyThreadState *ts, _PyStackRef *self, PyObject *name, _PyStackRef *method); +// Like PyObject_GetAttr but returns a _PyStackRef. For types, this can +// return a deferred reference to reduce reference count contention. +PyAPI_FUNC(_PyStackRef) _PyObject_GetAttrStackRef(PyObject *obj, PyObject *name); + // Cache the provided init method in the specialization cache of type if the // provided type version matches the current version of the type. // @@ -929,7 +912,9 @@ PyAPI_FUNC(int) _PyObject_GetMethodStackRef(PyThreadState *ts, PyObject *obj, // deferred reference counting. // // Returns 1 if the value was cached or 0 otherwise. -extern int _PyType_CacheInitForSpecialization(PyHeapTypeObject *type, +// +// Exported for external JIT support +PyAPI_FUNC(int) _PyType_CacheInitForSpecialization(PyHeapTypeObject *type, PyObject *init, unsigned int tp_version); @@ -1031,7 +1016,8 @@ enum _PyAnnotateFormat { _Py_ANNOTATE_FORMAT_STRING = 4, }; -int _PyObject_SetDict(PyObject *obj, PyObject *value); +extern int _PyObject_SetDict(PyObject *obj, PyObject *value); +extern int _PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict); #ifndef Py_GIL_DISABLED static inline Py_ALWAYS_INLINE void _Py_INCREF_MORTAL(PyObject *op) @@ -1045,8 +1031,14 @@ static inline Py_ALWAYS_INLINE void _Py_INCREF_MORTAL(PyObject *op) } #endif } +#else +# define _Py_INCREF_MORTAL(op) Py_INCREF(op) #endif +/* Utility for the tp_traverse slot of mutable heap types that have no other + * references. */ +PyAPI_FUNC(int) _PyObject_VisitType(PyObject *op, visitproc visit, void *arg); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h index a7ba8f34073..f6b4bd90a23 100644 --- a/Include/internal/pycore_obmalloc.h +++ b/Include/internal/pycore_obmalloc.h @@ -14,6 +14,12 @@ typedef unsigned int pymem_uint; /* assuming >= 16 bits */ #undef uint #define uint pymem_uint +/* NOTE: the following overviews were in the initial checkin, in 1998. In + * 2026, they're still helpful, but some details have changed. For example, + * we now use 32 size classes 16 bytes apart, and an arena is generally at + * least 1MB. Use sys._debugmallocstats() to see exact current details for + * the specific version of CPython used. + */ /* An object allocator for Python. @@ -208,7 +214,11 @@ typedef unsigned int pymem_uint; /* assuming >= 16 bits */ * mappings to reduce heap fragmentation. */ #ifdef USE_LARGE_ARENAS -#define ARENA_BITS 20 /* 1 MiB */ +# ifdef PYMALLOC_USE_HUGEPAGES +# define ARENA_BITS 21 /* 2 MiB */ +# else +# define ARENA_BITS 20 /* 1 MiB */ +# endif #else #define ARENA_BITS 18 /* 256 KiB */ #endif @@ -469,7 +479,7 @@ nfp free pools in usable_arenas. */ /* How many arena_objects do we initially allocate? - * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4MB before growing the + * 16 = can allocate 16 arenas = 16 * ARENA_SIZE before growing the * `arenas` vector. */ #define INITIAL_ARENA_OBJECTS 16 @@ -512,7 +522,11 @@ struct _obmalloc_mgmt { memory address bit allocation for keys - 64-bit pointers, IGNORE_BITS=0 and 2^20 arena size: + ARENA_BITS is configurable: 20 (1 MiB) by default on 64-bit, or + 21 (2 MiB) when PYMALLOC_USE_HUGEPAGES is enabled. All bit widths + below are derived from ARENA_BITS automatically. + + 64-bit pointers, IGNORE_BITS=0 and 2^20 arena size (default): 15 -> MAP_TOP_BITS 15 -> MAP_MID_BITS 14 -> MAP_BOT_BITS @@ -520,6 +534,14 @@ struct _obmalloc_mgmt { ---- 64 + 64-bit pointers, IGNORE_BITS=0 and 2^21 arena size (hugepages): + 15 -> MAP_TOP_BITS + 15 -> MAP_MID_BITS + 13 -> MAP_BOT_BITS + 21 -> ideal aligned arena + ---- + 64 + 64-bit pointers, IGNORE_BITS=16, and 2^20 arena size: 16 -> IGNORE_BITS 10 -> MAP_TOP_BITS @@ -675,7 +697,11 @@ struct _obmalloc_state { /* Allocate memory directly from the O/S virtual memory system, - * where supported. Otherwise fallback on malloc */ + * where supported. Otherwise fallback on malloc. + * + * Large-page and huge-page backends may round the mapped size up + * internally, so pass the original requested size back to + * _PyObject_VirtualFree(). */ void *_PyObject_VirtualAlloc(size_t size); void _PyObject_VirtualFree(void *, size_t size); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index bd6b84ec7fd..4f8b48db23d 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -66,6 +66,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2; case BINARY_OP_SUBSCR_TUPLE_INT: return 2; + case BINARY_OP_SUBSCR_USTR_INT: + return 2; case BINARY_OP_SUBTRACT_FLOAT: return 2; case BINARY_OP_SUBTRACT_INT: @@ -106,6 +108,10 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + oparg; case CALL_BUILTIN_O: return 2 + oparg; + case CALL_EX_NON_PY_GENERAL: + return 4; + case CALL_EX_PY: + return 4; case CALL_FUNCTION_EX: return 4; case CALL_INTRINSIC_1: @@ -151,7 +157,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case CHECK_EXC_MATCH: return 2; case CLEANUP_THROW: - return 3; + return 4; case COMPARE_OP: return 2; case COMPARE_OP_FLOAT: @@ -193,7 +199,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case END_FOR: return 1; case END_SEND: - return 2; + return 3; case ENTER_EXECUTOR: return 0; case EXIT_INIT_CHECK: @@ -214,6 +220,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2; case FOR_ITER_TUPLE: return 2; + case FOR_ITER_VIRTUAL: + return 2; case GET_AITER: return 1; case GET_ANEXT: @@ -222,9 +230,11 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case GET_ITER: return 1; - case GET_LEN: + case GET_ITER_SELF: return 1; - case GET_YIELD_FROM_ITER: + case GET_ITER_VIRTUAL: + return 1; + case GET_LEN: return 1; case IMPORT_FROM: return 1; @@ -241,7 +251,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case INSTRUMENTED_END_FOR: return 3; case INSTRUMENTED_END_SEND: - return 2; + return 3; case INSTRUMENTED_FOR_ITER: return 2; case INSTRUMENTED_INSTRUCTION: @@ -420,14 +430,16 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case RESUME_CHECK: return 0; + case RESUME_CHECK_JIT: + return 0; case RETURN_GENERATOR: return 0; case RETURN_VALUE: return 1; case SEND: - return 2; + return 3; case SEND_GEN: - return 2; + return 3; case SETUP_ANNOTATIONS: return 0; case SETUP_CLEANUP: @@ -488,6 +500,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case TO_BOOL_STR: return 1; + case TRACE_RECORD: + return 0; case UNARY_INVERT: return 1; case UNARY_NEGATIVE: @@ -532,7 +546,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case BINARY_OP_EXTEND: return 1; case BINARY_OP_INPLACE_ADD_UNICODE: - return 0; + return 1; case BINARY_OP_MULTIPLY_FLOAT: return 1; case BINARY_OP_MULTIPLY_INT: @@ -549,6 +563,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case BINARY_OP_SUBSCR_TUPLE_INT: return 1; + case BINARY_OP_SUBSCR_USTR_INT: + return 1; case BINARY_OP_SUBTRACT_FLOAT: return 1; case BINARY_OP_SUBTRACT_INT: @@ -589,6 +605,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case CALL_BUILTIN_O: return 1; + case CALL_EX_NON_PY_GENERAL: + return 1; + case CALL_EX_PY: + return 0; case CALL_FUNCTION_EX: return 1; case CALL_INTRINSIC_1: @@ -608,7 +628,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CALL_LEN: return 1; case CALL_LIST_APPEND: - return 0; + return 1; case CALL_METHOD_DESCRIPTOR_FAST: return 1; case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: @@ -634,7 +654,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CHECK_EXC_MATCH: return 2; case CLEANUP_THROW: - return 2; + return 3; case COMPARE_OP: return 1; case COMPARE_OP_FLOAT: @@ -697,6 +717,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 3; case FOR_ITER_TUPLE: return 3; + case FOR_ITER_VIRTUAL: + return 3; case GET_AITER: return 1; case GET_ANEXT: @@ -705,10 +727,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case GET_ITER: return 2; + case GET_ITER_SELF: + return 2; + case GET_ITER_VIRTUAL: + return 2; case GET_LEN: return 2; - case GET_YIELD_FROM_ITER: - return 1; case IMPORT_FROM: return 2; case IMPORT_NAME: @@ -788,7 +812,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case LOAD_ATTR_CLASS_WITH_METACLASS_CHECK: return 1 + (oparg & 1); case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: - return 1; + return 0; case LOAD_ATTR_INSTANCE_VALUE: return 1 + (oparg & 1); case LOAD_ATTR_METHOD_LAZY_DICT: @@ -903,14 +927,16 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case RESUME_CHECK: return 0; + case RESUME_CHECK_JIT: + return 0; case RETURN_GENERATOR: return 1; case RETURN_VALUE: return 1; case SEND: - return 2; + return 3; case SEND_GEN: - return 1; + return 2; case SETUP_ANNOTATIONS: return 0; case SETUP_CLEANUP: @@ -971,6 +997,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case TO_BOOL_STR: return 1; + case TRACE_RECORD: + return 0; case UNARY_INVERT: return 1; case UNARY_NEGATIVE: @@ -1028,9 +1056,13 @@ enum InstructionFormat { #define HAS_ESCAPES_FLAG (512) #define HAS_EXIT_FLAG (1024) #define HAS_PURE_FLAG (2048) -#define HAS_ERROR_NO_POP_FLAG (4096) -#define HAS_NO_SAVE_IP_FLAG (8192) -#define HAS_PERIODIC_FLAG (16384) +#define HAS_SYNC_SP_FLAG (4096) +#define HAS_ERROR_NO_POP_FLAG (8192) +#define HAS_NO_SAVE_IP_FLAG (16384) +#define HAS_PERIODIC_FLAG (32768) +#define HAS_UNPREDICTABLE_JUMP_FLAG (65536) +#define HAS_NEEDS_GUARD_IP_FLAG (131072) +#define HAS_RECORDS_VALUE_FLAG (262144) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -1043,9 +1075,13 @@ enum InstructionFormat { #define OPCODE_HAS_ESCAPES(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ESCAPES_FLAG)) #define OPCODE_HAS_EXIT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_EXIT_FLAG)) #define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG)) +#define OPCODE_HAS_SYNC_SP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_SYNC_SP_FLAG)) #define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) #define OPCODE_HAS_NO_SAVE_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NO_SAVE_IP_FLAG)) #define OPCODE_HAS_PERIODIC(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PERIODIC_FLAG)) +#define OPCODE_HAS_UNPREDICTABLE_JUMP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_UNPREDICTABLE_JUMP_FLAG)) +#define OPCODE_HAS_NEEDS_GUARD_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NEEDS_GUARD_IP_FLAG)) +#define OPCODE_HAS_RECORDS_VALUE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_RECORDS_VALUE_FLAG)) #define OPARG_SIMPLE 0 #define OPARG_CACHE_1 1 @@ -1062,27 +1098,28 @@ enum InstructionFormat { struct opcode_metadata { uint8_t valid_entry; uint8_t instr_format; - uint16_t flags; + uint32_t flags; }; -extern const struct opcode_metadata _PyOpcode_opcode_metadata[267]; +PyAPI_DATA(const struct opcode_metadata) _PyOpcode_opcode_metadata[267]; #ifdef NEED_OPCODE_METADATA const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { - [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, - [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_EXTEND] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, + [BINARY_OP_EXTEND] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, - [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG }, - [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_LIST_SLICE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_SUBSCR_LIST_SLICE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, + [BINARY_OP_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_SUBSCR_USTR_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_INTERPOLATION] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1094,34 +1131,36 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BUILD_TEMPLATE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [CACHE] = { true, INSTR_FMT_IX, 0 }, - [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ISINSTANCE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_EX_NON_PY_GENERAL] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_EX_PY] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_ISINSTANCE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LEN] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_LEN] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1129,9 +1168,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, - [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1141,9 +1180,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1151,30 +1190,32 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_VIRTUAL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [GET_ITER_SELF] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, + [GET_ITER_VIRTUAL] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_NOT_TAKEN] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_POP_ITER] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, @@ -1182,9 +1223,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1193,21 +1234,21 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [JUMP_BACKWARD_NO_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, @@ -1226,14 +1267,14 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SMALL_INT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_SPECIAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SPECIAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, @@ -1251,20 +1292,21 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, 0 }, - [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [RESUME] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [RESUME_CHECK] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [RESUME_CHECK_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG }, [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG }, @@ -1277,22 +1319,23 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, - [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, - [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TRACE_RECORD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_NOT] = { true, INSTR_FMT_IX, 0 }, [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [ANNOTATIONS_PLACEHOLDER] = { true, -1, HAS_PURE_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1307,7 +1350,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { }; #endif -#define MAX_UOP_PER_EXPANSION 10 +#define MAX_UOP_PER_EXPANSION 11 struct opcode_macro_expansion { int nuops; struct { int16_t uop; int8_t size; int8_t offset; } uops[MAX_UOP_PER_EXPANSION]; @@ -1317,22 +1360,23 @@ extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[256]; #ifdef NEED_OPCODE_METADATA const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { - [BINARY_OP] = { .nuops = 1, .uops = { { _BINARY_OP, OPARG_SIMPLE, 4 } } }, - [BINARY_OP_ADD_FLOAT] = { .nuops = 3, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_FLOAT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_ADD_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_INT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_ADD_UNICODE] = { .nuops = 3, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_UNICODE, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_EXTEND] = { .nuops = 2, .uops = { { _GUARD_BINARY_OP_EXTEND, 4, 1 }, { _BINARY_OP_EXTEND, 4, 1 } } }, + [BINARY_OP] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _BINARY_OP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, + [BINARY_OP_ADD_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_ADD_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_ADD_UNICODE] = { .nuops = 5, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_UNICODE, OPARG_SIMPLE, 5 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 5 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_EXTEND] = { .nuops = 4, .uops = { { _GUARD_BINARY_OP_EXTEND, 4, 1 }, { _BINARY_OP_EXTEND, 4, 1 }, { _POP_TOP, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, [BINARY_OP_INPLACE_ADD_UNICODE] = { .nuops = 3, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_INPLACE_ADD_UNICODE, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_MULTIPLY_FLOAT] = { .nuops = 3, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_MULTIPLY_FLOAT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_MULTIPLY_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_MULTIPLY_INT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_DICT] = { .nuops = 2, .uops = { { _GUARD_NOS_DICT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_DICT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_GETITEM] = { .nuops = 4, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 5 }, { _BINARY_OP_SUBSCR_CHECK_FUNC, OPARG_SIMPLE, 5 }, { _BINARY_OP_SUBSCR_INIT_CALL, OPARG_SIMPLE, 5 }, { _PUSH_FRAME, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_LIST_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_INT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_LIST_SLICE] = { .nuops = 3, .uops = { { _GUARD_TOS_SLICE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_SLICE, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_STR_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_STR_INT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_TUPLE_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_TUPLE, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_TUPLE_INT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 3, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBTRACT_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_INT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_MULTIPLY_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_MULTIPLY_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_MULTIPLY_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_MULTIPLY_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_DICT] = { .nuops = 4, .uops = { { _GUARD_NOS_ANY_DICT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_DICT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_GETITEM] = { .nuops = 5, .uops = { { _RECORD_NOS, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 5 }, { _BINARY_OP_SUBSCR_CHECK_FUNC, OPARG_SIMPLE, 5 }, { _BINARY_OP_SUBSCR_INIT_CALL, OPARG_SIMPLE, 5 }, { _PUSH_FRAME, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_LIST_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_LIST_SLICE] = { .nuops = 5, .uops = { { _GUARD_TOS_SLICE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_SLICE, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_STR_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_COMPACT_ASCII, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_STR_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_TUPLE_INT] = { .nuops = 6, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_TUPLE, OPARG_SIMPLE, 0 }, { _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_TUPLE_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_USTR_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_USTR_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBTRACT_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, [BINARY_SLICE] = { .nuops = 1, .uops = { { _BINARY_SLICE, OPARG_SIMPLE, 0 } } }, [BUILD_INTERPOLATION] = { .nuops = 1, .uops = { { _BUILD_INTERPOLATION, OPARG_SIMPLE, 0 } } }, [BUILD_LIST] = { .nuops = 1, .uops = { { _BUILD_LIST, OPARG_SIMPLE, 0 } } }, @@ -1342,40 +1386,42 @@ _PyOpcode_macro_expansion[256] = { [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, OPARG_SIMPLE, 0 } } }, [BUILD_TEMPLATE] = { .nuops = 1, .uops = { { _BUILD_TEMPLATE, OPARG_SIMPLE, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, OPARG_SIMPLE, 0 } } }, - [CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 4, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_AND_ALLOCATE_OBJECT, 2, 1 }, { _CREATE_INIT_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 10, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_BOUND_METHOD_GENERAL] = { .nuops = 7, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_BUILTIN_FAST] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_BUILTIN_O] = { .nuops = 2, .uops = { { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_1, OPARG_SIMPLE, 0 } } }, - [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, OPARG_SIMPLE, 0 } } }, + [CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_OBJECT, 2, 1 }, { _ALLOCATE_OBJECT, OPARG_SIMPLE, 3 }, { _CREATE_INIT_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 11, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_BOUND_METHOD_GENERAL] = { .nuops = 8, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_BUILTIN_CLASS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_FAST] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_O] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_O, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_EX_NON_PY_GENERAL] = { .nuops = 4, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CALL_FUNCTION_EX_NON_PY_GENERAL, OPARG_SIMPLE, 1 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 1 } } }, + [CALL_EX_PY] = { .nuops = 7, .uops = { { _RECORD_4OS, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CHECK_IS_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _PY_FRAME_EX, OPARG_SIMPLE, 1 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, + [CALL_INTRINSIC_1] = { .nuops = 2, .uops = { { _CALL_INTRINSIC_1, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [CALL_INTRINSIC_2] = { .nuops = 3, .uops = { { _CALL_INTRINSIC_2, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [CALL_ISINSTANCE] = { .nuops = 3, .uops = { { _GUARD_THIRD_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_ISINSTANCE, OPARG_SIMPLE, 3 }, { _CALL_ISINSTANCE, OPARG_SIMPLE, 3 } } }, - [CALL_KW_BOUND_METHOD] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_KW_BOUND_METHOD] = { .nuops = 7, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, OPARG_SIMPLE, 3 }, { _CALL_KW_NON_PY, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_KW_PY] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_LEN] = { .nuops = 3, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_LEN, OPARG_SIMPLE, 3 }, { _CALL_LEN, OPARG_SIMPLE, 3 } } }, - [CALL_LIST_APPEND] = { .nuops = 4, .uops = { { _GUARD_CALLABLE_LIST_APPEND, OPARG_SIMPLE, 3 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 3 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 3 }, { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_NON_PY_GENERAL] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE, OPARG_SIMPLE, 3 }, { _CALL_NON_PY_GENERAL, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_PY_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_PY_GENERAL] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_STR_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_STR_1, OPARG_SIMPLE, 3 }, { _CALL_STR_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_TUPLE_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TUPLE_1, OPARG_SIMPLE, 3 }, { _CALL_TUPLE_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_TYPE_1] = { .nuops = 3, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TYPE_1, OPARG_SIMPLE, 3 }, { _CALL_TYPE_1, OPARG_SIMPLE, 3 } } }, + [CALL_KW_PY] = { .nuops = 7, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_LEN] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_LEN, OPARG_SIMPLE, 3 }, { _CALL_LEN, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, + [CALL_LIST_APPEND] = { .nuops = 6, .uops = { { _GUARD_CALLABLE_LIST_APPEND, OPARG_SIMPLE, 3 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 3 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 3 }, { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, + [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 5, .uops = { { _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 8, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_NON_PY_GENERAL] = { .nuops = 4, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_IS_NOT_PY_CALLABLE, OPARG_SIMPLE, 3 }, { _CALL_NON_PY_GENERAL, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_PY_EXACT_ARGS] = { .nuops = 9, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_PY_GENERAL] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_STR_1] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_STR_1, OPARG_SIMPLE, 3 }, { _CALL_STR_1, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_TUPLE_1] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TUPLE_1, OPARG_SIMPLE, 3 }, { _CALL_TUPLE_1, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_TYPE_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TYPE_1, OPARG_SIMPLE, 3 }, { _CALL_TYPE_1, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { _CHECK_EG_MATCH, OPARG_SIMPLE, 0 } } }, [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { _CHECK_EXC_MATCH, OPARG_SIMPLE, 0 } } }, [COMPARE_OP] = { .nuops = 1, .uops = { { _COMPARE_OP, OPARG_SIMPLE, 0 } } }, - [COMPARE_OP_FLOAT] = { .nuops = 3, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _COMPARE_OP_FLOAT, OPARG_SIMPLE, 1 } } }, - [COMPARE_OP_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _COMPARE_OP_INT, OPARG_SIMPLE, 1 } } }, - [COMPARE_OP_STR] = { .nuops = 3, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _COMPARE_OP_STR, OPARG_SIMPLE, 1 } } }, - [CONTAINS_OP] = { .nuops = 1, .uops = { { _CONTAINS_OP, OPARG_SIMPLE, 0 } } }, - [CONTAINS_OP_DICT] = { .nuops = 2, .uops = { { _GUARD_TOS_DICT, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_DICT, OPARG_SIMPLE, 1 } } }, - [CONTAINS_OP_SET] = { .nuops = 2, .uops = { { _GUARD_TOS_ANY_SET, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_SET, OPARG_SIMPLE, 1 } } }, + [COMPARE_OP_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _COMPARE_OP_FLOAT, OPARG_SIMPLE, 1 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 1 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 1 } } }, + [COMPARE_OP_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _COMPARE_OP_INT, OPARG_SIMPLE, 1 }, { _POP_TOP_INT, OPARG_SIMPLE, 1 }, { _POP_TOP_INT, OPARG_SIMPLE, 1 } } }, + [COMPARE_OP_STR] = { .nuops = 5, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _COMPARE_OP_STR, OPARG_SIMPLE, 1 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 1 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 1 } } }, + [CONTAINS_OP] = { .nuops = 3, .uops = { { _CONTAINS_OP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [CONTAINS_OP_DICT] = { .nuops = 4, .uops = { { _GUARD_TOS_ANY_DICT, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_DICT, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, + [CONTAINS_OP_SET] = { .nuops = 4, .uops = { { _GUARD_TOS_ANY_SET, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_SET, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, [CONVERT_VALUE] = { .nuops = 1, .uops = { { _CONVERT_VALUE, OPARG_SIMPLE, 0 } } }, [COPY] = { .nuops = 1, .uops = { { _COPY, OPARG_SIMPLE, 0 } } }, [COPY_FREE_VARS] = { .nuops = 1, .uops = { { _COPY_FREE_VARS, OPARG_SIMPLE, 0 } } }, @@ -1385,42 +1431,48 @@ _PyOpcode_macro_expansion[256] = { [DELETE_GLOBAL] = { .nuops = 1, .uops = { { _DELETE_GLOBAL, OPARG_SIMPLE, 0 } } }, [DELETE_NAME] = { .nuops = 1, .uops = { { _DELETE_NAME, OPARG_SIMPLE, 0 } } }, [DELETE_SUBSCR] = { .nuops = 1, .uops = { { _DELETE_SUBSCR, OPARG_SIMPLE, 0 } } }, - [DICT_MERGE] = { .nuops = 1, .uops = { { _DICT_MERGE, OPARG_SIMPLE, 0 } } }, - [DICT_UPDATE] = { .nuops = 1, .uops = { { _DICT_UPDATE, OPARG_SIMPLE, 0 } } }, + [DICT_MERGE] = { .nuops = 2, .uops = { { _DICT_MERGE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [DICT_UPDATE] = { .nuops = 2, .uops = { { _DICT_UPDATE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [END_FOR] = { .nuops = 1, .uops = { { _END_FOR, OPARG_SIMPLE, 0 } } }, [END_SEND] = { .nuops = 1, .uops = { { _END_SEND, OPARG_SIMPLE, 0 } } }, [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { _EXIT_INIT_CHECK, OPARG_SIMPLE, 0 } } }, [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { _FORMAT_SIMPLE, OPARG_SIMPLE, 0 } } }, [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { _FORMAT_WITH_SPEC, OPARG_SIMPLE, 0 } } }, [FOR_ITER] = { .nuops = 1, .uops = { { _FOR_ITER, OPARG_REPLACED, 0 } } }, - [FOR_ITER_GEN] = { .nuops = 3, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _FOR_ITER_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, + [FOR_ITER_GEN] = { .nuops = 4, .uops = { { _RECORD_NOS_GEN_FUNC, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _FOR_ITER_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, [FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, OPARG_SIMPLE, 1 }, { _ITER_JUMP_LIST, OPARG_REPLACED, 1 }, { _ITER_NEXT_LIST, OPARG_REPLACED, 1 } } }, [FOR_ITER_RANGE] = { .nuops = 3, .uops = { { _ITER_CHECK_RANGE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_RANGE, OPARG_REPLACED, 1 }, { _ITER_NEXT_RANGE, OPARG_SIMPLE, 1 } } }, [FOR_ITER_TUPLE] = { .nuops = 3, .uops = { { _ITER_CHECK_TUPLE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_TUPLE, OPARG_REPLACED, 1 }, { _ITER_NEXT_TUPLE, OPARG_SIMPLE, 1 } } }, + [FOR_ITER_VIRTUAL] = { .nuops = 2, .uops = { { _GUARD_NOS_ITER_VIRTUAL, OPARG_SIMPLE, 1 }, { _FOR_ITER_VIRTUAL, OPARG_REPLACED, 1 } } }, [GET_AITER] = { .nuops = 1, .uops = { { _GET_AITER, OPARG_SIMPLE, 0 } } }, [GET_ANEXT] = { .nuops = 1, .uops = { { _GET_ANEXT, OPARG_SIMPLE, 0 } } }, [GET_AWAITABLE] = { .nuops = 1, .uops = { { _GET_AWAITABLE, OPARG_SIMPLE, 0 } } }, - [GET_ITER] = { .nuops = 1, .uops = { { _GET_ITER, OPARG_SIMPLE, 0 } } }, + [GET_ITER] = { .nuops = 2, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _GET_ITER, OPARG_SIMPLE, 0 } } }, + [GET_ITER_SELF] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _GUARD_ITERATOR, OPARG_SIMPLE, 1 }, { _PUSH_NULL, OPARG_SIMPLE, 1 } } }, + [GET_ITER_VIRTUAL] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _GUARD_ITER_VIRTUAL, OPARG_SIMPLE, 1 }, { _PUSH_TAGGED_ZERO, OPARG_SIMPLE, 1 } } }, [GET_LEN] = { .nuops = 1, .uops = { { _GET_LEN, OPARG_SIMPLE, 0 } } }, - [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { _GET_YIELD_FROM_ITER, OPARG_SIMPLE, 0 } } }, [IMPORT_FROM] = { .nuops = 1, .uops = { { _IMPORT_FROM, OPARG_SIMPLE, 0 } } }, [IMPORT_NAME] = { .nuops = 1, .uops = { { _IMPORT_NAME, OPARG_SIMPLE, 0 } } }, - [IS_OP] = { .nuops = 1, .uops = { { _IS_OP, OPARG_SIMPLE, 0 } } }, + [IS_OP] = { .nuops = 3, .uops = { { _IS_OP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [JUMP_BACKWARD] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, + [JUMP_BACKWARD_NO_INTERRUPT] = { .nuops = 1, .uops = { { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 0 } } }, + [JUMP_BACKWARD_NO_JIT] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, OPARG_SIMPLE, 0 } } }, - [LIST_EXTEND] = { .nuops = 1, .uops = { { _LIST_EXTEND, OPARG_SIMPLE, 0 } } }, + [LIST_EXTEND] = { .nuops = 2, .uops = { { _LIST_EXTEND, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, OPARG_SIMPLE, 8 } } }, - [LOAD_ATTR_CLASS] = { .nuops = 3, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { .nuops = 4, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _GUARD_TYPE_VERSION, 2, 3 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, - [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, OPARG_SIMPLE, 3 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, - [LOAD_ATTR_MODULE] = { .nuops = 3, .uops = { { _LOAD_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, OPERAND1_1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, OPARG_SIMPLE, 3 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, - [LOAD_ATTR_PROPERTY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_PROPERTY_FRAME, 4, 5 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 9 }, { _PUSH_FRAME, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_SLOT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_WITH_HINT, 1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_CLASS] = { .nuops = 4, .uops = { { _RECORD_TOS, OPARG_SIMPLE, 1 }, { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { .nuops = 5, .uops = { { _RECORD_TOS, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_CLASS, 2, 3 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { .nuops = 7, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME, 2, 3 }, { _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME, OPERAND1_4, 5 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 9 }, { _PUSH_FRAME, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 6, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, + [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, OPARG_SIMPLE, 3 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_MODULE] = { .nuops = 4, .uops = { { _LOAD_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, OPERAND1_1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, OPARG_SIMPLE, 3 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_PROPERTY] = { .nuops = 7, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_PROPERTY_FRAME, 2, 3 }, { _LOAD_ATTR_PROPERTY_FRAME, OPERAND1_4, 5 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 9 }, { _PUSH_FRAME, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_SLOT] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_WITH_HINT] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_WITH_HINT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { _LOAD_BUILD_CLASS, OPARG_SIMPLE, 0 } } }, [LOAD_COMMON_CONSTANT] = { .nuops = 1, .uops = { { _LOAD_COMMON_CONSTANT, OPARG_SIMPLE, 0 } } }, [LOAD_CONST] = { .nuops = 1, .uops = { { _LOAD_CONST, OPARG_SIMPLE, 0 } } }, @@ -1438,13 +1490,13 @@ _PyOpcode_macro_expansion[256] = { [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, OPARG_SIMPLE, 0 } } }, [LOAD_NAME] = { .nuops = 1, .uops = { { _LOAD_NAME, OPARG_SIMPLE, 0 } } }, [LOAD_SMALL_INT] = { .nuops = 1, .uops = { { _LOAD_SMALL_INT, OPARG_SIMPLE, 0 } } }, - [LOAD_SPECIAL] = { .nuops = 2, .uops = { { _INSERT_NULL, OPARG_SIMPLE, 0 }, { _LOAD_SPECIAL, OPARG_SIMPLE, 0 } } }, + [LOAD_SPECIAL] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _INSERT_NULL, OPARG_SIMPLE, 0 }, { _LOAD_SPECIAL, OPARG_SIMPLE, 0 } } }, [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_ATTR, OPARG_SIMPLE, 1 } } }, - [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_METHOD, OPARG_SIMPLE, 1 } } }, + [LOAD_SUPER_ATTR_METHOD] = { .nuops = 3, .uops = { { _RECORD_NOS, OPARG_SIMPLE, 0 }, { _GUARD_LOAD_SUPER_ATTR_METHOD, OPARG_SIMPLE, 1 }, { _LOAD_SUPER_ATTR_METHOD, OPARG_SIMPLE, 1 } } }, [MAKE_CELL] = { .nuops = 1, .uops = { { _MAKE_CELL, OPARG_SIMPLE, 0 } } }, - [MAKE_FUNCTION] = { .nuops = 1, .uops = { { _MAKE_FUNCTION, OPARG_SIMPLE, 0 } } }, + [MAKE_FUNCTION] = { .nuops = 2, .uops = { { _MAKE_FUNCTION, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [MAP_ADD] = { .nuops = 1, .uops = { { _MAP_ADD, OPARG_SIMPLE, 0 } } }, - [MATCH_CLASS] = { .nuops = 1, .uops = { { _MATCH_CLASS, OPARG_SIMPLE, 0 } } }, + [MATCH_CLASS] = { .nuops = 4, .uops = { { _MATCH_CLASS, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [MATCH_KEYS] = { .nuops = 1, .uops = { { _MATCH_KEYS, OPARG_SIMPLE, 0 } } }, [MATCH_MAPPING] = { .nuops = 1, .uops = { { _MATCH_MAPPING, OPARG_SIMPLE, 0 } } }, [MATCH_SEQUENCE] = { .nuops = 1, .uops = { { _MATCH_SEQUENCE, OPARG_SIMPLE, 0 } } }, @@ -1459,38 +1511,38 @@ _PyOpcode_macro_expansion[256] = { [POP_TOP] = { .nuops = 1, .uops = { { _POP_TOP, OPARG_SIMPLE, 0 } } }, [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { _PUSH_EXC_INFO, OPARG_SIMPLE, 0 } } }, [PUSH_NULL] = { .nuops = 1, .uops = { { _PUSH_NULL, OPARG_SIMPLE, 0 } } }, - [RESUME_CHECK] = { .nuops = 1, .uops = { { _RESUME_CHECK, OPARG_SIMPLE, 0 } } }, + [RESUME_CHECK] = { .nuops = 1, .uops = { { _RESUME_CHECK, OPARG_SIMPLE, 1 } } }, [RETURN_GENERATOR] = { .nuops = 1, .uops = { { _RETURN_GENERATOR, OPARG_SIMPLE, 0 } } }, - [RETURN_VALUE] = { .nuops = 1, .uops = { { _RETURN_VALUE, OPARG_SIMPLE, 0 } } }, - [SEND_GEN] = { .nuops = 3, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _SEND_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, + [RETURN_VALUE] = { .nuops = 2, .uops = { { _MAKE_HEAP_SAFE, OPARG_SIMPLE, 0 }, { _RETURN_VALUE, OPARG_SIMPLE, 0 } } }, + [SEND_GEN] = { .nuops = 4, .uops = { { _RECORD_3OS_GEN_FUNC, OPARG_SIMPLE, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _SEND_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { _SETUP_ANNOTATIONS, OPARG_SIMPLE, 0 } } }, [SET_ADD] = { .nuops = 1, .uops = { { _SET_ADD, OPARG_SIMPLE, 0 } } }, [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, OPARG_SIMPLE, 0 } } }, - [SET_UPDATE] = { .nuops = 1, .uops = { { _SET_UPDATE, OPARG_SIMPLE, 0 } } }, + [SET_UPDATE] = { .nuops = 2, .uops = { { _SET_UPDATE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, OPARG_SIMPLE, 3 } } }, - [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION_AND_LOCK, 2, 1 }, { _GUARD_DORV_NO_DICT, OPARG_SIMPLE, 3 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, - [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, - [STORE_ATTR_WITH_HINT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_WITH_HINT, 1, 3 } } }, + [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 6, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _LOCK_OBJECT, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION_LOCKED, 2, 1 }, { _GUARD_DORV_NO_DICT, OPARG_SIMPLE, 3 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, + [STORE_ATTR_SLOT] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, + [STORE_ATTR_WITH_HINT] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_WITH_HINT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, [STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, OPARG_SIMPLE, 0 } } }, - [STORE_FAST] = { .nuops = 1, .uops = { { _STORE_FAST, OPARG_SIMPLE, 0 } } }, - [STORE_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { _STORE_FAST, OPARG_TOP, 0 }, { _LOAD_FAST, OPARG_BOTTOM, 0 } } }, - [STORE_FAST_STORE_FAST] = { .nuops = 2, .uops = { { _STORE_FAST, OPARG_TOP, 0 }, { _STORE_FAST, OPARG_BOTTOM, 0 } } }, + [STORE_FAST] = { .nuops = 2, .uops = { { _SWAP_FAST, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [STORE_FAST_LOAD_FAST] = { .nuops = 3, .uops = { { _SWAP_FAST, OPARG_TOP, 0 }, { _POP_TOP, OPARG_TOP, 0 }, { _LOAD_FAST, OPARG_BOTTOM, 0 } } }, + [STORE_FAST_STORE_FAST] = { .nuops = 4, .uops = { { _SWAP_FAST, OPARG_TOP, 0 }, { _POP_TOP, OPARG_TOP, 0 }, { _SWAP_FAST, OPARG_BOTTOM, 0 }, { _POP_TOP, OPARG_BOTTOM, 0 } } }, [STORE_GLOBAL] = { .nuops = 1, .uops = { { _STORE_GLOBAL, OPARG_SIMPLE, 0 } } }, [STORE_NAME] = { .nuops = 1, .uops = { { _STORE_NAME, OPARG_SIMPLE, 0 } } }, [STORE_SLICE] = { .nuops = 1, .uops = { { _STORE_SLICE, OPARG_SIMPLE, 0 } } }, [STORE_SUBSCR] = { .nuops = 1, .uops = { { _STORE_SUBSCR, OPARG_SIMPLE, 0 } } }, - [STORE_SUBSCR_DICT] = { .nuops = 2, .uops = { { _GUARD_NOS_DICT, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_DICT, OPARG_SIMPLE, 1 } } }, - [STORE_SUBSCR_LIST_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_LIST_INT, OPARG_SIMPLE, 1 } } }, + [STORE_SUBSCR_DICT] = { .nuops = 3, .uops = { { _GUARD_NOS_DICT, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_DICT, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, + [STORE_SUBSCR_LIST_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_LIST_INT, OPARG_SIMPLE, 1 }, { _POP_TOP_INT, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, [SWAP] = { .nuops = 1, .uops = { { _SWAP, OPARG_SIMPLE, 0 } } }, [TO_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL, OPARG_SIMPLE, 2 } } }, - [TO_BOOL_ALWAYS_TRUE] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _REPLACE_WITH_TRUE, OPARG_SIMPLE, 3 } } }, + [TO_BOOL_ALWAYS_TRUE] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _REPLACE_WITH_TRUE, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL_BOOL, OPARG_SIMPLE, 3 } } }, - [TO_BOOL_INT] = { .nuops = 1, .uops = { { _TO_BOOL_INT, OPARG_SIMPLE, 3 } } }, - [TO_BOOL_LIST] = { .nuops = 2, .uops = { { _GUARD_TOS_LIST, OPARG_SIMPLE, 0 }, { _TO_BOOL_LIST, OPARG_SIMPLE, 3 } } }, + [TO_BOOL_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _TO_BOOL_INT, OPARG_SIMPLE, 3 }, { _POP_TOP_INT, OPARG_SIMPLE, 3 } } }, + [TO_BOOL_LIST] = { .nuops = 3, .uops = { { _GUARD_TOS_LIST, OPARG_SIMPLE, 0 }, { _TO_BOOL_LIST, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [TO_BOOL_NONE] = { .nuops = 1, .uops = { { _TO_BOOL_NONE, OPARG_SIMPLE, 3 } } }, - [TO_BOOL_STR] = { .nuops = 2, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _TO_BOOL_STR, OPARG_SIMPLE, 3 } } }, - [UNARY_INVERT] = { .nuops = 1, .uops = { { _UNARY_INVERT, OPARG_SIMPLE, 0 } } }, - [UNARY_NEGATIVE] = { .nuops = 1, .uops = { { _UNARY_NEGATIVE, OPARG_SIMPLE, 0 } } }, + [TO_BOOL_STR] = { .nuops = 3, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _TO_BOOL_STR, OPARG_SIMPLE, 3 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 3 } } }, + [UNARY_INVERT] = { .nuops = 2, .uops = { { _UNARY_INVERT, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [UNARY_NEGATIVE] = { .nuops = 2, .uops = { { _UNARY_NEGATIVE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [UNARY_NOT] = { .nuops = 1, .uops = { { _UNARY_NOT, OPARG_SIMPLE, 0 } } }, [UNPACK_EX] = { .nuops = 1, .uops = { { _UNPACK_EX, OPARG_SIMPLE, 0 } } }, [UNPACK_SEQUENCE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE, OPARG_SIMPLE, 0 } } }, @@ -1498,11 +1550,11 @@ _PyOpcode_macro_expansion[256] = { [UNPACK_SEQUENCE_TUPLE] = { .nuops = 2, .uops = { { _GUARD_TOS_TUPLE, OPARG_SIMPLE, 0 }, { _UNPACK_SEQUENCE_TUPLE, OPARG_SIMPLE, 1 } } }, [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 2, .uops = { { _GUARD_TOS_TUPLE, OPARG_SIMPLE, 0 }, { _UNPACK_SEQUENCE_TWO_TUPLE, OPARG_SIMPLE, 1 } } }, [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { _WITH_EXCEPT_START, OPARG_SIMPLE, 0 } } }, - [YIELD_VALUE] = { .nuops = 1, .uops = { { _YIELD_VALUE, OPARG_SIMPLE, 0 } } }, + [YIELD_VALUE] = { .nuops = 2, .uops = { { _MAKE_HEAP_SAFE, OPARG_SIMPLE, 0 }, { _YIELD_VALUE, OPARG_SIMPLE, 0 } } }, }; #endif // NEED_OPCODE_METADATA -extern const char *_PyOpcode_OpName[267]; +PyAPI_DATA(const char) *_PyOpcode_OpName[267]; #ifdef NEED_OPCODE_METADATA const char *_PyOpcode_OpName[267] = { [ANNOTATIONS_PLACEHOLDER] = "ANNOTATIONS_PLACEHOLDER", @@ -1520,6 +1572,7 @@ const char *_PyOpcode_OpName[267] = { [BINARY_OP_SUBSCR_LIST_SLICE] = "BINARY_OP_SUBSCR_LIST_SLICE", [BINARY_OP_SUBSCR_STR_INT] = "BINARY_OP_SUBSCR_STR_INT", [BINARY_OP_SUBSCR_TUPLE_INT] = "BINARY_OP_SUBSCR_TUPLE_INT", + [BINARY_OP_SUBSCR_USTR_INT] = "BINARY_OP_SUBSCR_USTR_INT", [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", [BINARY_SLICE] = "BINARY_SLICE", @@ -1540,6 +1593,8 @@ const char *_PyOpcode_OpName[267] = { [CALL_BUILTIN_FAST] = "CALL_BUILTIN_FAST", [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [CALL_BUILTIN_O] = "CALL_BUILTIN_O", + [CALL_EX_NON_PY_GENERAL] = "CALL_EX_NON_PY_GENERAL", + [CALL_EX_PY] = "CALL_EX_PY", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", [CALL_INTRINSIC_1] = "CALL_INTRINSIC_1", [CALL_INTRINSIC_2] = "CALL_INTRINSIC_2", @@ -1594,12 +1649,14 @@ const char *_PyOpcode_OpName[267] = { [FOR_ITER_LIST] = "FOR_ITER_LIST", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", + [FOR_ITER_VIRTUAL] = "FOR_ITER_VIRTUAL", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", [GET_AWAITABLE] = "GET_AWAITABLE", [GET_ITER] = "GET_ITER", + [GET_ITER_SELF] = "GET_ITER_SELF", + [GET_ITER_VIRTUAL] = "GET_ITER_VIRTUAL", [GET_LEN] = "GET_LEN", - [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [IMPORT_FROM] = "IMPORT_FROM", [IMPORT_NAME] = "IMPORT_NAME", [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", @@ -1697,6 +1754,7 @@ const char *_PyOpcode_OpName[267] = { [RESERVED] = "RESERVED", [RESUME] = "RESUME", [RESUME_CHECK] = "RESUME_CHECK", + [RESUME_CHECK_JIT] = "RESUME_CHECK_JIT", [RETURN_GENERATOR] = "RETURN_GENERATOR", [RETURN_VALUE] = "RETURN_VALUE", [SEND] = "SEND", @@ -1731,6 +1789,7 @@ const char *_PyOpcode_OpName[267] = { [TO_BOOL_LIST] = "TO_BOOL_LIST", [TO_BOOL_NONE] = "TO_BOOL_NONE", [TO_BOOL_STR] = "TO_BOOL_STR", + [TRACE_RECORD] = "TRACE_RECORD", [UNARY_INVERT] = "UNARY_INVERT", [UNARY_NEGATIVE] = "UNARY_NEGATIVE", [UNARY_NOT] = "UNARY_NOT", @@ -1744,9 +1803,10 @@ const char *_PyOpcode_OpName[267] = { }; #endif -extern const uint8_t _PyOpcode_Caches[256]; +PyAPI_DATA(const uint8_t) _PyOpcode_Caches[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Caches[256] = { + [RESUME] = 1, [TO_BOOL] = 3, [STORE_SUBSCR] = 1, [SEND] = 1, @@ -1762,16 +1822,19 @@ const uint8_t _PyOpcode_Caches[256] = { [POP_JUMP_IF_FALSE] = 1, [POP_JUMP_IF_NONE] = 1, [POP_JUMP_IF_NOT_NONE] = 1, + [GET_ITER] = 1, [FOR_ITER] = 1, [CALL] = 3, [CALL_KW] = 3, + [CALL_FUNCTION_EX] = 1, [BINARY_OP] = 5, }; #endif -extern const uint8_t _PyOpcode_Deopt[256]; +PyAPI_DATA(const uint8_t) _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { + [120] = 120, [121] = 121, [122] = 122, [123] = 123, @@ -1779,13 +1842,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [125] = 125, [126] = 126, [127] = 127, - [210] = 210, - [211] = 211, - [212] = 212, - [213] = 213, - [214] = 214, - [215] = 215, - [216] = 216, [217] = 217, [218] = 218, [219] = 219, @@ -1802,7 +1858,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [230] = 230, [231] = 231, [232] = 232, - [233] = 233, [BINARY_OP] = BINARY_OP, [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, @@ -1817,6 +1872,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [BINARY_OP_SUBSCR_LIST_SLICE] = BINARY_OP, [BINARY_OP_SUBSCR_STR_INT] = BINARY_OP, [BINARY_OP_SUBSCR_TUPLE_INT] = BINARY_OP, + [BINARY_OP_SUBSCR_USTR_INT] = BINARY_OP, [BINARY_OP_SUBTRACT_FLOAT] = BINARY_OP, [BINARY_OP_SUBTRACT_INT] = BINARY_OP, [BINARY_SLICE] = BINARY_SLICE, @@ -1837,6 +1893,8 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_BUILTIN_FAST] = CALL, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, [CALL_BUILTIN_O] = CALL, + [CALL_EX_NON_PY_GENERAL] = CALL_FUNCTION_EX, + [CALL_EX_PY] = CALL_FUNCTION_EX, [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, [CALL_INTRINSIC_1] = CALL_INTRINSIC_1, [CALL_INTRINSIC_2] = CALL_INTRINSIC_2, @@ -1891,12 +1949,14 @@ const uint8_t _PyOpcode_Deopt[256] = { [FOR_ITER_LIST] = FOR_ITER, [FOR_ITER_RANGE] = FOR_ITER, [FOR_ITER_TUPLE] = FOR_ITER, + [FOR_ITER_VIRTUAL] = FOR_ITER, [GET_AITER] = GET_AITER, [GET_ANEXT] = GET_ANEXT, [GET_AWAITABLE] = GET_AWAITABLE, [GET_ITER] = GET_ITER, + [GET_ITER_SELF] = GET_ITER, + [GET_ITER_VIRTUAL] = GET_ITER, [GET_LEN] = GET_LEN, - [GET_YIELD_FROM_ITER] = GET_YIELD_FROM_ITER, [IMPORT_FROM] = IMPORT_FROM, [IMPORT_NAME] = IMPORT_NAME, [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, @@ -1988,6 +2048,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [RESERVED] = RESERVED, [RESUME] = RESUME, [RESUME_CHECK] = RESUME, + [RESUME_CHECK_JIT] = RESUME, [RETURN_GENERATOR] = RETURN_GENERATOR, [RETURN_VALUE] = RETURN_VALUE, [SEND] = SEND, @@ -2018,6 +2079,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [TO_BOOL_LIST] = TO_BOOL, [TO_BOOL_NONE] = TO_BOOL, [TO_BOOL_STR] = TO_BOOL, + [TRACE_RECORD] = TRACE_RECORD, [UNARY_INVERT] = UNARY_INVERT, [UNARY_NEGATIVE] = UNARY_NEGATIVE, [UNARY_NOT] = UNARY_NOT, @@ -2033,6 +2095,7 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ + case 120: \ case 121: \ case 122: \ case 123: \ @@ -2040,13 +2103,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 125: \ case 126: \ case 127: \ - case 210: \ - case 211: \ - case 212: \ - case 213: \ - case 214: \ - case 215: \ - case 216: \ case 217: \ case 218: \ case 219: \ @@ -2063,7 +2119,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 230: \ case 231: \ case 232: \ - case 233: \ ; struct pseudo_targets { uint8_t as_sequence; diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index 79a1a242556..3e2c4ae411c 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -73,16 +73,28 @@ extern "C" { #define CONSTANT_BUILTIN_TUPLE 2 #define CONSTANT_BUILTIN_ALL 3 #define CONSTANT_BUILTIN_ANY 4 -#define NUM_COMMON_CONSTANTS 5 +#define CONSTANT_BUILTIN_LIST 5 +#define CONSTANT_BUILTIN_SET 6 +#define CONSTANT_NONE 7 +#define CONSTANT_EMPTY_STR 8 +#define CONSTANT_TRUE 9 +#define CONSTANT_FALSE 10 +#define CONSTANT_MINUS_ONE 11 +#define NUM_COMMON_CONSTANTS 12 /* Values used in the oparg for RESUME */ #define RESUME_AT_FUNC_START 0 #define RESUME_AFTER_YIELD 1 #define RESUME_AFTER_YIELD_FROM 2 #define RESUME_AFTER_AWAIT 3 +#define RESUME_AT_GEN_EXPR_START 4 -#define RESUME_OPARG_LOCATION_MASK 0x3 -#define RESUME_OPARG_DEPTH1_MASK 0x4 +#define RESUME_OPARG_LOCATION_MASK 0x7 +#define RESUME_OPARG_DEPTH1_MASK 0x8 + +#define GET_ITER_YIELD_FROM 1 +#define GET_ITER_YIELD_FROM_NO_CHECK 2 +#define GET_ITER_YIELD_FROM_CORO_CHECK 3 #ifdef __cplusplus } diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 94d01999f68..a0727c045e5 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -9,67 +9,183 @@ extern "C" { #endif #include "pycore_typedefs.h" // _PyInterpreterFrame +#include "pycore_uop.h" // _PyUOpInstruction #include "pycore_uop_ids.h" #include "pycore_stackref.h" // _PyStackRef +#include "pycore_optimizer_types.h" #include +/* Fitness controls how long a trace can grow. + * Starts at FITNESS_INITIAL, then decreases from per-bytecode buffer usage + * plus branch/frame heuristics. The trace stops when fitness drops below the + * current exit_quality. + * + * Design targets for the constants below: + * 1. Reaching the abstract frame-depth limit should drop fitness below + * EXIT_QUALITY_SPECIALIZABLE. + * 2. A backward edge should leave budget for roughly N_BACKWARD_SLACK more + * bytecodes, assuming AVG_SLOTS_PER_INSTRUCTION. + * 3. Roughly seven balanced branches should reduce fitness to + * EXIT_QUALITY_DEFAULT after per-slot costs. + * 4. A push followed by a matching return is net-zero on frame-specific + * fitness, excluding per-slot costs. + */ +#define MAX_TARGET_LENGTH (UOP_MAX_TRACE_LENGTH / 2) +#define OPTIMIZER_EFFECTIVENESS 2 +#define FITNESS_INITIAL (MAX_TARGET_LENGTH * OPTIMIZER_EFFECTIVENESS) + +/* Exit quality thresholds: trace stops when fitness < exit_quality. + * Higher = trace is more willing to stop here. */ +#define EXIT_QUALITY_CLOSE_LOOP (FITNESS_INITIAL - AVG_SLOTS_PER_INSTRUCTION*4) +#define EXIT_QUALITY_ENTER_EXECUTOR (FITNESS_INITIAL * 1 / 8) +#define EXIT_QUALITY_DEFAULT (FITNESS_INITIAL / 40) +#define EXIT_QUALITY_SPECIALIZABLE (FITNESS_INITIAL / 80) + +/* Estimated buffer slots per bytecode, used only to derive heuristics. + * Runtime charging uses trace-buffer capacity consumed for each bytecode. */ +#define AVG_SLOTS_PER_INSTRUCTION 6 + +/* Heuristic backward-edge exit quality: leave room for about 1 unroll and + * N_BACKWARD_SLACK more bytecodes before reaching EXIT_QUALITY_CLOSE_LOOP, + * based on AVG_SLOTS_PER_INSTRUCTION. */ +#define N_BACKWARD_SLACK 10 +#define EXIT_QUALITY_BACKWARD_EDGE (EXIT_QUALITY_CLOSE_LOOP / 2 - N_BACKWARD_SLACK * AVG_SLOTS_PER_INSTRUCTION) + +/* Penalty for a balanced branch. + * It is sized so repeated balanced branches can drive a trace toward + * EXIT_QUALITY_DEFAULT, while compute_branch_penalty() keeps any single branch + * from dominating the budget. + */ +#define FITNESS_BRANCH_BALANCED ((FITNESS_INITIAL - EXIT_QUALITY_DEFAULT - \ + (MAX_TARGET_LENGTH / 14 * AVG_SLOTS_PER_INSTRUCTION)) / (14)) + + +typedef struct _PyJitUopBuffer { + _PyUOpInstruction *start; + _PyUOpInstruction *next; + _PyUOpInstruction *end; +} _PyJitUopBuffer; + +typedef struct _JitOptRefBuffer { + JitOptRef *used; + JitOptRef *end; +} _JitOptRefBuffer; + +typedef struct _JitOptContext { + char done; + char out_of_space; + bool contradiction; + // Has the builtins dict been watched? + bool builtins_watched; + // The current "executing" frame. + _Py_UOpsAbstractFrame *frame; + _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH]; + int curr_frame_depth; + + // Arena for the symbolic types. + ty_arena t_arena; + + /* To do -- We could make this more space efficient + * by using a single array and growing the stack and + * locals toward each other. */ + _JitOptRefBuffer locals; + _JitOptRefBuffer stack; + JitOptRef locals_array[ABSTRACT_INTERP_LOCALS_SIZE]; + JitOptRef stack_array[ABSTRACT_INTERP_STACK_SIZE]; + _PyJitUopBuffer out_buffer; + _PyBloomFilter *dependencies; +} JitOptContext; + + +static inline void +uop_buffer_init(_PyJitUopBuffer *trace, _PyUOpInstruction *start, uint32_t size) +{ + trace->next = trace->start = start; + trace->end = start + size; +} + +static inline _PyUOpInstruction * +uop_buffer_last(_PyJitUopBuffer *trace) +{ + assert(trace->next > trace->start); + return trace->next-1; +} + +static inline int +uop_buffer_length(_PyJitUopBuffer *trace) +{ + return (int)(trace->next - trace->start); +} + +static inline int +uop_buffer_remaining_space(_PyJitUopBuffer *trace) +{ + return (int)(trace->end - trace->next); +} + +typedef struct _PyJitTracerInitialState { + int stack_depth; + int chain_depth; + struct _PyExitData *exit; + PyCodeObject *code; // Strong + PyFunctionObject *func; // Strong + struct _PyExecutorObject *executor; // Strong + _Py_CODEUNIT *start_instr; + _Py_CODEUNIT *close_loop_instr; + _Py_CODEUNIT *jump_backward_instr; +} _PyJitTracerInitialState; + +#define MAX_RECORDED_VALUES 3 +typedef struct _PyJitTracerPreviousState { + int instr_oparg; + int instr_stacklevel; + _Py_CODEUNIT *instr; + PyCodeObject *instr_code; // Strong + struct _PyInterpreterFrame *instr_frame; + PyObject *recorded_values[MAX_RECORDED_VALUES]; // Strong, may be NULL + int recorded_count; +} _PyJitTracerPreviousState; + +typedef struct _PyJitTracerTranslatorState { + int32_t fitness; // Current trace fitness, starts high, decrements + int frame_depth; // Current inline depth (0 = root frame) +} _PyJitTracerTranslatorState; + +typedef struct _PyJitTracerState { + bool is_tracing; + _PyJitTracerInitialState initial_state; + _PyJitTracerPreviousState prev_state; + _PyJitTracerTranslatorState translator_state; + JitOptContext opt_context; + _PyJitUopBuffer code_buffer; + _PyJitUopBuffer out_buffer; + _PyUOpInstruction uop_array[2 * UOP_MAX_TRACE_LENGTH]; +} _PyJitTracerState; typedef struct _PyExecutorLinkListNode { struct _PyExecutorObject *next; struct _PyExecutorObject *previous; } _PyExecutorLinkListNode; - -/* Bloom filter with m = 256 - * https://en.wikipedia.org/wiki/Bloom_filter */ -#define _Py_BLOOM_FILTER_WORDS 8 - -typedef struct { - uint32_t bits[_Py_BLOOM_FILTER_WORDS]; -} _PyBloomFilter; - typedef struct { uint8_t opcode; uint8_t oparg; - uint8_t valid:1; - uint8_t linked:1; - uint8_t chain_depth:6; // Must be big enough for MAX_CHAIN_DEPTH - 1. - bool warm; - int index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). - _PyBloomFilter bloom; - _PyExecutorLinkListNode links; + uint8_t valid; + uint8_t chain_depth; // Must be big enough for MAX_CHAIN_DEPTH - 1. + bool cold; + uint8_t pending_deletion; + int32_t index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). + int32_t bloom_array_idx; // Index in interp->executor_blooms/executor_ptrs. + _PyExecutorLinkListNode links; // Used by deletion list. PyCodeObject *code; // Weak (NULL if no corresponding ENTER_EXECUTOR). } _PyVMData; -/* Depending on the format, - * the 32 bits between the oparg and operand are: - * UOP_FORMAT_TARGET: - * uint32_t target; - * UOP_FORMAT_JUMP - * uint16_t jump_target; - * uint16_t error_target; - */ -typedef struct { - uint16_t opcode:15; - uint16_t format:1; - uint16_t oparg; - union { - uint32_t target; - struct { - uint16_t jump_target; - uint16_t error_target; - }; - }; - uint64_t operand0; // A cache entry - uint64_t operand1; -#ifdef Py_STATS - uint64_t execution_count; -#endif -} _PyUOpInstruction; - typedef struct _PyExitData { uint32_t target; - uint16_t index; + uint16_t index:12; + uint16_t stack_cache:2; + uint16_t is_dynamic:1; + uint16_t is_control_flow:1; _Py_BackoffCounter temperature; struct _PyExecutorObject *executor; } _PyExitData; @@ -82,23 +198,96 @@ typedef struct _PyExecutorObject { uint32_t code_size; size_t jit_size; void *jit_code; + void *jit_gdb_handle; _PyExitData exits[1]; } _PyExecutorObject; -/* If pending deletion list gets large enough, then scan, - * and free any executors that aren't executing - * i.e. any that aren't a thread's current_executor. */ -#define EXECUTOR_DELETE_LIST_MAX 100 - // Export for '_opcode' shared extension (JIT compiler). PyAPI_FUNC(_PyExecutorObject*) _Py_GetExecutor(PyCodeObject *code, int offset); -void _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); +int _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); void _Py_ExecutorDetach(_PyExecutorObject *); -void _Py_BloomFilter_Init(_PyBloomFilter *); -void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); +/* We use a bloomfilter with k = 6, m = 256 + * The choice of k and the following constants + * could do with a more rigorous analysis, + * but here is a simple analysis: + * + * We want to keep the false positive rate low. + * For n = 5 (a trace depends on 5 objects), + * we expect 30 bits set, giving a false positive + * rate of (30/256)**6 == 2.5e-6 which is plenty + * good enough. + * + * However with n = 10 we expect 60 bits set (worst case), + * giving a false positive of (60/256)**6 == 0.0001 + * + * We choose k = 6, rather than a higher number as + * it means the false positive rate grows slower for high n. + * + * n = 5, k = 6 => fp = 2.6e-6 + * n = 5, k = 8 => fp = 3.5e-7 + * n = 10, k = 6 => fp = 1.6e-4 + * n = 10, k = 8 => fp = 0.9e-4 + * n = 15, k = 6 => fp = 0.18% + * n = 15, k = 8 => fp = 0.23% + * n = 20, k = 6 => fp = 1.1% + * n = 20, k = 8 => fp = 2.3% + * + * The above analysis assumes perfect hash functions, + * but those don't exist, so the real false positive + * rates may be worse. + */ + +#define _Py_BLOOM_FILTER_K 6 +#define _Py_BLOOM_FILTER_SEED 20221211 + +static inline uint64_t +address_to_hash(void *ptr) { + assert(ptr != NULL); + uint64_t uhash = _Py_BLOOM_FILTER_SEED; + uintptr_t addr = (uintptr_t)ptr; + for (int i = 0; i < SIZEOF_VOID_P; i++) { + uhash ^= addr & 255; + uhash *= (uint64_t)PyHASH_MULTIPLIER; + addr >>= 8; + } + return uhash; +} + +static inline void +_Py_BloomFilter_Init(_PyBloomFilter *bloom) +{ + for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { + bloom->bits[i] = 0; + } +} + +static inline void +_Py_BloomFilter_Add(_PyBloomFilter *bloom, void *ptr) +{ + uint64_t hash = address_to_hash(ptr); + assert(_Py_BLOOM_FILTER_K <= 8); + for (int i = 0; i < _Py_BLOOM_FILTER_K; i++) { + uint8_t bits = hash & 255; + bloom->bits[bits >> _Py_BLOOM_FILTER_WORD_SHIFT] |= + (_Py_bloom_filter_word_t)1 << (bits & (_Py_BLOOM_FILTER_BITS_PER_WORD - 1)); + hash >>= 8; + } +} + +static inline bool +bloom_filter_may_contain(const _PyBloomFilter *bloom, const _PyBloomFilter *hashes) +{ + for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { + if ((bloom->bits[i] & hashes->bits[i]) != hashes->bits[i]) { + return false; + } + } + return true; +} + #define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 #define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6 @@ -110,22 +299,18 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); #else # define _Py_Executors_InvalidateDependency(A, B, C) ((void)0) # define _Py_Executors_InvalidateAll(A, B) ((void)0) -# define _Py_Executors_InvalidateCold(A) ((void)0) #endif // Used as the threshold to trigger executor invalidation when -// trace_run_counter is greater than this value. -#define JIT_CLEANUP_THRESHOLD 100000 +// executor_creation_counter is greater than this value. +// This value is arbitrary and was not optimized. +#define JIT_CLEANUP_THRESHOLD 1000 -// This is the length of the trace we project initially. -#define UOP_MAX_TRACE_LENGTH 1200 - -#define TRACE_STACK_SIZE 5 - -int _Py_uop_analyze_and_optimize(_PyInterpreterFrame *frame, - _PyUOpInstruction *trace, int trace_len, int curr_stackentries, - _PyBloomFilter *dependencies); +int _Py_uop_analyze_and_optimize( + _PyThreadStateImpl *tstate, + _PyUOpInstruction *input, int trace_len, int curr_stackentries, + _PyUOpInstruction *output, _PyBloomFilter *dependencies); extern PyTypeObject _PyUOpExecutor_Type; @@ -151,90 +336,16 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) return inst->error_target; } -// Holds locals, stack, locals, stack ... co_consts (in that order) -#define MAX_ABSTRACT_INTERP_SIZE 4096 - -#define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * 5) - -// Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) -#define MAX_ABSTRACT_FRAME_DEPTH (TRACE_STACK_SIZE + 2) - -// The maximum number of side exits that we can take before requiring forward -// progress (and inserting a new ENTER_EXECUTOR instruction). In practice, this -// is the "maximum amount of polymorphism" that an isolated trace tree can -// handle before rejoining the rest of the program. -#define MAX_CHAIN_DEPTH 4 - -/* Symbols */ -/* See explanation in optimizer_symbols.c */ - - -typedef enum _JitSymType { - JIT_SYM_UNKNOWN_TAG = 1, - JIT_SYM_NULL_TAG = 2, - JIT_SYM_NON_NULL_TAG = 3, - JIT_SYM_BOTTOM_TAG = 4, - JIT_SYM_TYPE_VERSION_TAG = 5, - JIT_SYM_KNOWN_CLASS_TAG = 6, - JIT_SYM_KNOWN_VALUE_TAG = 7, - JIT_SYM_TUPLE_TAG = 8, - JIT_SYM_TRUTHINESS_TAG = 9, - JIT_SYM_COMPACT_INT = 10, -} JitSymType; - -typedef struct _jit_opt_known_class { - uint8_t tag; - uint32_t version; - PyTypeObject *type; -} JitOptKnownClass; - -typedef struct _jit_opt_known_version { - uint8_t tag; - uint32_t version; -} JitOptKnownVersion; - -typedef struct _jit_opt_known_value { - uint8_t tag; - PyObject *value; -} JitOptKnownValue; - -#define MAX_SYMBOLIC_TUPLE_SIZE 7 - -typedef struct _jit_opt_tuple { - uint8_t tag; - uint8_t length; - uint16_t items[MAX_SYMBOLIC_TUPLE_SIZE]; -} JitOptTuple; - -typedef struct { - uint8_t tag; - bool invert; - uint16_t value; -} JitOptTruthiness; - -typedef struct { - uint8_t tag; -} JitOptCompactInt; - -typedef union _jit_opt_symbol { - uint8_t tag; - JitOptKnownClass cls; - JitOptKnownValue value; - JitOptKnownVersion version; - JitOptTuple tuple; - JitOptTruthiness truthiness; - JitOptCompactInt compact; -} JitOptSymbol; - - -// This mimics the _PyStackRef API -typedef union { - uintptr_t bits; -} JitOptRef; #define REF_IS_BORROWED 1 +#define REF_IS_UNIQUE 2 +#define REF_IS_INVALID 3 +#define REF_TAG_BITS 3 -#define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~REF_IS_BORROWED))) +#define REF_GET_TAG(x) ((uintptr_t)(x) & (REF_TAG_BITS)) +#define REF_CLEAR_TAG(x) ((uintptr_t)(x) & (~REF_TAG_BITS)) + +#define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~REF_TAG_BITS))) static inline JitOptSymbol * PyJitRef_Unwrap(JitOptRef ref) @@ -251,16 +362,58 @@ PyJitRef_Wrap(JitOptSymbol *sym) return (JitOptRef){.bits=(uintptr_t)sym}; } +static inline JitOptRef +PyJitRef_WrapInvalid(void *ptr) +{ + return (JitOptRef){.bits = REF_CLEAR_TAG((uintptr_t)ptr) | REF_IS_INVALID}; +} + +static inline bool +PyJitRef_IsInvalid(JitOptRef ref) +{ + return REF_GET_TAG(ref.bits) == REF_IS_INVALID; +} + +static inline JitOptRef +PyJitRef_MakeUnique(JitOptRef ref) +{ + return (JitOptRef){ REF_CLEAR_TAG(ref.bits) | REF_IS_UNIQUE }; +} + +static inline bool +PyJitRef_IsUnique(JitOptRef ref) +{ + return REF_GET_TAG(ref.bits) == REF_IS_UNIQUE; +} + +static inline JitOptRef +PyJitRef_StripBorrowInfo(JitOptRef ref) +{ + if (PyJitRef_IsUnique(ref)) { + return ref; + } + return (JitOptRef){ .bits = REF_CLEAR_TAG(ref.bits) }; +} + static inline JitOptRef PyJitRef_StripReferenceInfo(JitOptRef ref) { return PyJitRef_Wrap(PyJitRef_Unwrap(ref)); } +static inline JitOptRef +PyJitRef_RemoveUnique(JitOptRef ref) +{ + if (PyJitRef_IsUnique(ref)) { + ref = PyJitRef_StripReferenceInfo(ref); + } + return ref; +} + static inline JitOptRef PyJitRef_Borrow(JitOptRef ref) { - return (JitOptRef){ .bits = ref.bits | REF_IS_BORROWED }; + return (JitOptRef){ .bits = REF_CLEAR_TAG(ref.bits) | REF_IS_BORROWED }; } static const JitOptRef PyJitRef_NULL = {.bits = REF_IS_BORROWED}; @@ -274,44 +427,9 @@ PyJitRef_IsNull(JitOptRef ref) static inline int PyJitRef_IsBorrowed(JitOptRef ref) { - return (ref.bits & REF_IS_BORROWED) == REF_IS_BORROWED; + return REF_GET_TAG(ref.bits) == REF_IS_BORROWED; } -struct _Py_UOpsAbstractFrame { - // Max stacklen - int stack_len; - int locals_len; - - JitOptRef *stack_pointer; - JitOptRef *stack; - JitOptRef *locals; -}; - -typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; - -typedef struct ty_arena { - int ty_curr_number; - int ty_max_number; - JitOptSymbol arena[TY_ARENA_SIZE]; -} ty_arena; - -typedef struct _JitOptContext { - char done; - char out_of_space; - bool contradiction; - // The current "executing" frame. - _Py_UOpsAbstractFrame *frame; - _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH]; - int curr_frame_depth; - - // Arena for the symbolic types. - ty_arena t_arena; - - JitOptRef *n_consumed; - JitOptRef *limit; - JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; -} JitOptContext; - extern bool _Py_uop_sym_is_null(JitOptRef sym); extern bool _Py_uop_sym_is_not_null(JitOptRef sym); extern bool _Py_uop_sym_is_const(JitOptContext *ctx, JitOptRef sym); @@ -324,11 +442,13 @@ extern JitOptRef _Py_uop_sym_new_type( extern JitOptRef _Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val); extern JitOptRef _Py_uop_sym_new_const_steal(JitOptContext *ctx, PyObject *const_val); bool _Py_uop_sym_is_safe_const(JitOptContext *ctx, JitOptRef sym); +bool _Py_uop_sym_is_not_container(JitOptRef sym); _PyStackRef _Py_uop_sym_get_const_as_stackref(JitOptContext *ctx, JitOptRef sym); extern JitOptRef _Py_uop_sym_new_null(JitOptContext *ctx); extern bool _Py_uop_sym_has_type(JitOptRef sym); extern bool _Py_uop_sym_matches_type(JitOptRef sym, PyTypeObject *typ); extern bool _Py_uop_sym_matches_type_version(JitOptRef sym, unsigned int version); +extern unsigned int _Py_uop_sym_get_type_version(JitOptRef sym); extern void _Py_uop_sym_set_null(JitOptContext *ctx, JitOptRef sym); extern void _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptRef sym); extern void _Py_uop_sym_set_type(JitOptContext *ctx, JitOptRef sym, PyTypeObject *typ); @@ -344,21 +464,36 @@ extern JitOptRef _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef value, extern bool _Py_uop_sym_is_compact_int(JitOptRef sym); extern JitOptRef _Py_uop_sym_new_compact_int(JitOptContext *ctx); extern void _Py_uop_sym_set_compact_int(JitOptContext *ctx, JitOptRef sym); +extern JitOptRef _Py_uop_sym_new_predicate(JitOptContext *ctx, JitOptRef lhs_ref, JitOptRef rhs_ref, JitOptPredicateKind kind); +extern void _Py_uop_sym_apply_predicate_narrowing(JitOptContext *ctx, JitOptRef sym, bool branch_is_true); +extern void _Py_uop_sym_set_recorded_value(JitOptContext *ctx, JitOptRef sym, PyObject *value); +extern void _Py_uop_sym_set_recorded_type(JitOptContext *ctx, JitOptRef sym, PyTypeObject *type); +extern void _Py_uop_sym_set_recorded_gen_func(JitOptContext *ctx, JitOptRef ref, PyFunctionObject *value); +extern PyCodeObject *_Py_uop_sym_get_probable_func_code(JitOptRef sym); +extern PyObject *_Py_uop_sym_get_probable_value(JitOptRef sym); +extern PyTypeObject *_Py_uop_sym_get_probable_type(JitOptRef sym); +extern JitOptRef *_Py_uop_sym_set_stack_depth(JitOptContext *ctx, int stack_depth, JitOptRef *current_sp); -extern void _Py_uop_abstractcontext_init(JitOptContext *ctx); +extern void _Py_uop_abstractcontext_init(JitOptContext *ctx, _PyBloomFilter *dependencies); extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( JitOptContext *ctx, PyCodeObject *co, - int curr_stackentries, JitOptRef *args, int arg_len); -extern int _Py_uop_frame_pop(JitOptContext *ctx); + +extern _Py_UOpsAbstractFrame *_Py_uop_frame_new_from_symbol( + JitOptContext *ctx, + JitOptRef callable, + JitOptRef *args, + int arg_len); + +extern int _Py_uop_frame_pop(JitOptContext *ctx, PyCodeObject *co); PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); -PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyExecutorObject **exec_ptr, int chain_depth); +PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, PyThreadState *tstate); static inline _PyExecutorObject *_PyExecutor_FromExit(_PyExitData *exit) { @@ -367,23 +502,54 @@ static inline _PyExecutorObject *_PyExecutor_FromExit(_PyExitData *exit) } extern _PyExecutorObject *_PyExecutor_GetColdExecutor(void); +extern _PyExecutorObject *_PyExecutor_GetColdDynamicExecutor(void); PyAPI_FUNC(void) _PyExecutor_ClearExit(_PyExitData *exit); -static inline int is_terminator(const _PyUOpInstruction *uop) -{ - int opcode = uop->opcode; - return ( - opcode == _EXIT_TRACE || - opcode == _JUMP_TO_TOP - ); -} - extern void _PyExecutor_Free(_PyExecutorObject *self); PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); #ifdef _Py_TIER2 -extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); +PyAPI_FUNC(void) _Py_ClearExecutorDeletionList(PyInterpreterState *interp); +#endif + +PyAPI_FUNC(int) _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int stop_tracing_opcode); + +PyAPI_FUNC(int) +_PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *start_instr, + _Py_CODEUNIT *close_loop_instr, _PyStackRef *stack_pointer, int chain_depth, _PyExitData *exit, + int oparg, _PyExecutorObject *current_executor); + +PyAPI_FUNC(void) _PyJit_FinalizeTracing(PyThreadState *tstate, int err); +PyAPI_FUNC(bool) _PyJit_EnterExecutorShouldStopTracing(int og_opcode); + +void _PyPrintExecutor(_PyExecutorObject *executor, const _PyUOpInstruction *marker); +void _PyJit_TracerFree(_PyThreadStateImpl *_tstate); + +#ifdef _Py_TIER2 +typedef void (*_Py_RecordFuncPtr)(_PyInterpreterFrame *frame, _PyStackRef *stackpointer, int oparg, PyObject **recorded_value); +PyAPI_DATA(const _Py_RecordFuncPtr) _PyOpcode_RecordFunctions[]; + +typedef struct { + uint8_t count; + uint8_t indices[MAX_RECORDED_VALUES]; +} _PyOpcodeRecordEntry; + +typedef struct { + uint8_t count; + uint8_t transform_mask; + uint8_t slots[MAX_RECORDED_VALUES]; +} _PyOpcodeRecordSlotMap; + +PyAPI_DATA(const _PyOpcodeRecordEntry) _PyOpcode_RecordEntries[256]; +PyAPI_DATA(const _PyOpcodeRecordSlotMap) _PyOpcode_RecordSlotMaps[256]; + +/* Convert a family-recorded value to the form a recorder uop expects. + * If no transform is needed, return the input value unchanged. + * Takes ownership of `value` and returns a new strong reference or NULL. + */ +PyAPI_FUNC(PyObject *) _PyOpcode_RecordTransformValue(int uop, PyObject *value); #endif #ifdef __cplusplus diff --git a/Include/internal/pycore_optimizer_types.h b/Include/internal/pycore_optimizer_types.h new file mode 100644 index 00000000000..a722652cc81 --- /dev/null +++ b/Include/internal/pycore_optimizer_types.h @@ -0,0 +1,169 @@ +#ifndef Py_INTERNAL_OPTIMIZER_TYPES_H +#define Py_INTERNAL_OPTIMIZER_TYPES_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include +#include "pycore_uop.h" // UOP_MAX_TRACE_LENGTH + +#define ABSTRACT_INTERP_STACK_SIZE 256 +#define ABSTRACT_INTERP_LOCALS_SIZE 512 + + +#define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * 5) + +// Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) +#define MAX_ABSTRACT_FRAME_DEPTH (16) + +// The maximum number of side exits that we can take before requiring forward +// progress (and inserting a new ENTER_EXECUTOR instruction). In practice, this +// is the "maximum amount of polymorphism" that an isolated trace tree can +// handle before rejoining the rest of the program. +#define MAX_CHAIN_DEPTH 4 + +/* Symbols */ +/* See explanation in optimizer_symbols.c */ + + +typedef enum _JitSymType { + JIT_SYM_UNKNOWN_TAG = 1, + JIT_SYM_NULL_TAG = 2, + JIT_SYM_NON_NULL_TAG = 3, + JIT_SYM_BOTTOM_TAG = 4, + JIT_SYM_TYPE_VERSION_TAG = 5, + JIT_SYM_KNOWN_CLASS_TAG = 7, + JIT_SYM_KNOWN_VALUE_TAG = 8, + JIT_SYM_TUPLE_TAG = 9, + JIT_SYM_TRUTHINESS_TAG = 10, + JIT_SYM_COMPACT_INT = 11, + JIT_SYM_PREDICATE_TAG = 12, + JIT_SYM_RECORDED_VALUE_TAG = 13, + JIT_SYM_RECORDED_TYPE_TAG = 14, + JIT_SYM_RECORDED_GEN_FUNC_TAG = 15, +} JitSymType; + +typedef struct _jit_opt_known_class { + uint8_t tag; + uint32_t version; + PyTypeObject *type; +} JitOptKnownClass; + +typedef struct _jit_opt_known_version { + uint8_t tag; + uint32_t version; +} JitOptKnownVersion; + +typedef struct _jit_opt_known_func_version { + uint8_t tag; + uint32_t func_version; +} JitOptKnownFuncVersion; + +typedef struct _jit_opt_known_value { + uint8_t tag; + PyObject *value; +} JitOptKnownValue; + +#define MAX_SYMBOLIC_TUPLE_SIZE 7 + +typedef struct _jit_opt_tuple { + uint8_t tag; + uint8_t length; + uint16_t items[MAX_SYMBOLIC_TUPLE_SIZE]; +} JitOptTuple; + +typedef struct { + uint8_t tag; + bool invert; + uint16_t value; +} JitOptTruthiness; + +typedef enum { + JIT_PRED_IS, + JIT_PRED_IS_NOT, + JIT_PRED_EQ, + JIT_PRED_NE, +} JitOptPredicateKind; + +typedef struct { + uint8_t tag; + uint8_t kind; + uint16_t lhs; + uint16_t rhs; +} JitOptPredicate; + +typedef struct _jit_opt_recorded_value { + uint8_t tag; + bool known_type; + PyObject *value; +} JitOptRecordedValue; + +typedef struct _jit_opt_recorded_type { + uint8_t tag; + PyTypeObject *type; +} JitOptRecordedType; + +/* Represents a generator, but we record the + * function as the generator is emphemeral */ +typedef struct _jit_opt_recorded_gen_func { + uint8_t tag; + PyFunctionObject *func; +} JitOptRecordedGenFunc; + +typedef struct { + uint8_t tag; +} JitOptCompactInt; + +typedef union _jit_opt_symbol { + uint8_t tag; + JitOptKnownClass cls; + JitOptKnownValue value; + JitOptKnownVersion version; + JitOptKnownFuncVersion func_version; + JitOptTuple tuple; + JitOptTruthiness truthiness; + JitOptCompactInt compact; + JitOptPredicate predicate; + JitOptRecordedValue recorded_value; + JitOptRecordedType recorded_type; + JitOptRecordedGenFunc recorded_gen_func; +} JitOptSymbol; + +// This mimics the _PyStackRef API +typedef union { + uintptr_t bits; +} JitOptRef; + +typedef struct _Py_UOpsAbstractFrame { + bool globals_watched; + // The version number of the globals dicts, once checked. 0 if unchecked. + uint32_t globals_checked_version; + // Max stacklen + int stack_len; + int locals_len; + bool caller; // We have made a call from this frame during the trace + bool is_c_recursion_checked; + JitOptRef callable; + PyFunctionObject *func; + PyCodeObject *code; + + JitOptRef *stack_pointer; + JitOptRef *stack; + JitOptRef *locals; +} _Py_UOpsAbstractFrame; + +typedef struct ty_arena { + int ty_curr_number; + int ty_max_number; + JitOptSymbol arena[TY_ARENA_SIZE]; +} ty_arena; + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_OPTIMIZER_TYPES_H */ diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index 2885dee63dc..b89d02035db 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -14,21 +14,6 @@ extern "C" { #include "pycore_pyarena.h" // PyArena _Py_DECLARE_STR(empty, "") -#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED) -#define _parser_runtime_state_INIT \ - { \ - .mutex = {0}, \ - .dummy_name = { \ - .kind = Name_kind, \ - .v.Name.id = &_Py_STR(empty), \ - .v.Name.ctx = Load, \ - .lineno = 1, \ - .col_offset = 0, \ - .end_lineno = 1, \ - .end_col_offset = 0, \ - }, \ - } -#else #define _parser_runtime_state_INIT \ { \ .dummy_name = { \ @@ -41,14 +26,14 @@ _Py_DECLARE_STR(empty, "") .end_col_offset = 0, \ }, \ } -#endif extern struct _mod* _PyParser_ASTFromString( const char *str, PyObject* filename, int mode, PyCompilerFlags *flags, - PyArena *arena); + PyArena *arena, + PyObject *module); extern struct _mod* _PyParser_ASTFromFile( FILE *fp, diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index c31c3365700..fafdd728a82 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -31,6 +31,8 @@ extern "C" { _Py_atomic_store_ptr(&value, new_value) #define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) \ _Py_atomic_load_ptr_acquire(&value) +#define FT_ATOMIC_LOAD_PTR_CONSUME(value) \ + _Py_atomic_load_ptr_consume(&value) #define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) \ _Py_atomic_load_uintptr_acquire(&value) #define FT_ATOMIC_LOAD_PTR_RELAXED(value) \ @@ -39,12 +41,18 @@ extern "C" { _Py_atomic_load_uint8(&value) #define FT_ATOMIC_STORE_UINT8(value, new_value) \ _Py_atomic_store_uint8(&value, new_value) +#define FT_ATOMIC_LOAD_INT8_RELAXED(value) \ + _Py_atomic_load_int8_relaxed(&value) #define FT_ATOMIC_LOAD_UINT8_RELAXED(value) \ _Py_atomic_load_uint8_relaxed(&value) #define FT_ATOMIC_LOAD_UINT16_RELAXED(value) \ _Py_atomic_load_uint16_relaxed(&value) #define FT_ATOMIC_LOAD_UINT32_RELAXED(value) \ _Py_atomic_load_uint32_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT64_ACQUIRE(value) \ + _Py_atomic_load_uint64_acquire(&value) +#define FT_ATOMIC_LOAD_UINT64_RELAXED(value) \ + _Py_atomic_load_uint64_relaxed(&value) #define FT_ATOMIC_LOAD_ULONG_RELAXED(value) \ _Py_atomic_load_ulong_relaxed(&value) #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ @@ -53,14 +61,26 @@ extern "C" { _Py_atomic_store_ptr_release(&value, new_value) #define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) \ _Py_atomic_store_uintptr_release(&value, new_value) +#define FT_ATOMIC_STORE_INT8_RELAXED(value, new_value) \ + _Py_atomic_store_int8_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_INT8_RELEASE(value, new_value) \ + _Py_atomic_store_int8_release(&value, new_value) #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \ _Py_atomic_store_ssize_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) \ + _Py_atomic_store_ssize_release(&value, new_value) #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) \ _Py_atomic_store_uint8_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) \ _Py_atomic_store_uint16_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \ _Py_atomic_store_uint32_relaxed(&value, new_value) +#define FT_ATOMIC_AND_UINT64(value, new_value) \ + (void)_Py_atomic_and_uint64(&value, new_value) +#define FT_ATOMIC_OR_UINT64(value, new_value) \ + (void)_Py_atomic_or_uint64(&value, new_value) +#define FT_ATOMIC_ADD_UINT64(value, new_value) \ + (void)_Py_atomic_add_uint64(&value, new_value) #define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) \ _Py_atomic_store_char_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) \ @@ -77,10 +97,16 @@ extern "C" { _Py_atomic_store_ushort_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_USHORT_RELAXED(value) \ _Py_atomic_load_ushort_relaxed(&value) +#define FT_ATOMIC_LOAD_INT(value) \ + _Py_atomic_load_int(&value) +#define FT_ATOMIC_STORE_INT(value, new_value) \ + _Py_atomic_store_int(&value, new_value) #define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) \ _Py_atomic_store_int_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_INT_RELAXED(value) \ _Py_atomic_load_int_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT(value) \ + _Py_atomic_load_uint(&value) #define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) \ _Py_atomic_store_uint_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_UINT_RELAXED(value) \ @@ -121,21 +147,31 @@ extern "C" { #define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) value #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value #define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_PTR_CONSUME(value) value #define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) value #define FT_ATOMIC_LOAD_PTR_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT8(value) value #define FT_ATOMIC_STORE_UINT8(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_INT8_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT8_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT16_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT32_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT64_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_UINT64_RELAXED(value) value #define FT_ATOMIC_LOAD_ULONG_RELAXED(value) value #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_INT8_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_INT8_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_AND_UINT64(value, new_value) (void)(value &= new_value) +#define FT_ATOMIC_OR_UINT64(value, new_value) (void)(value |= new_value) +#define FT_ATOMIC_ADD_UINT64(value, new_value) (void)(value += new_value) #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) value #define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) value @@ -144,8 +180,11 @@ extern "C" { #define FT_ATOMIC_STORE_SHORT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_USHORT_RELAXED(value) value #define FT_ATOMIC_STORE_USHORT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_INT(value) value +#define FT_ATOMIC_STORE_INT(value, new_value) value = new_value #define FT_ATOMIC_LOAD_INT_RELAXED(value) value #define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UINT(value) value #define FT_ATOMIC_LOAD_UINT_RELAXED(value) value #define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_LONG_RELAXED(value) value diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 2c2048f7e12..e436aa6bf12 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -11,8 +11,8 @@ extern "C" { /* Error handling definitions */ -extern _PyErr_StackItem* _PyErr_GetTopmostException(PyThreadState *tstate); -extern PyObject* _PyErr_GetHandledException(PyThreadState *); +PyAPI_FUNC(_PyErr_StackItem*) _PyErr_GetTopmostException(PyThreadState *tstate); +PyAPI_FUNC(PyObject*) _PyErr_GetHandledException(PyThreadState *); extern void _PyErr_SetHandledException(PyThreadState *, PyObject *); extern void _PyErr_GetExcInfo(PyThreadState *, PyObject **, PyObject **, PyObject **); @@ -29,7 +29,8 @@ PyAPI_FUNC(PyObject*) _PyErr_FormatFromCause( ... ); -extern int _PyException_AddNote( +// Export for 'pyexpat' shared extension. +PyAPI_FUNC(int) _PyException_AddNote( PyObject *exc, PyObject *note); @@ -108,7 +109,7 @@ extern void _PyErr_Restore( PyObject *value, PyObject *traceback); -extern void _PyErr_SetObject( +PyAPI_FUNC(void) _PyErr_SetObject( PyThreadState *tstate, PyObject *type, PyObject *value); @@ -123,7 +124,8 @@ extern void _PyErr_SetNone(PyThreadState *tstate, PyObject *exception); extern PyObject* _PyErr_NoMemory(PyThreadState *tstate); extern int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, - int end_lineno, int end_col_offset); + int end_lineno, int end_col_offset, + PyObject *module); extern void _PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset); @@ -168,7 +170,8 @@ extern PyObject* _PyErr_FormatFromCauseTstate( const char *format, ...); -extern PyObject* _PyExc_CreateExceptionGroup( +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyExc_CreateExceptionGroup( const char *msg, PyObject *excs); @@ -179,7 +182,8 @@ extern PyObject* _PyExc_PrepReraiseStar( extern int _PyErr_CheckSignalsTstate(PyThreadState *tstate); extern void _Py_DumpExtensionModules(int fd, PyInterpreterState *interp); -extern PyObject* _Py_CalculateSuggestions(PyObject *dir, PyObject *name); +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _Py_CalculateSuggestions(PyObject *dir, PyObject *name); extern PyObject* _Py_Offer_Suggestions(PyObject* exception); // Export for '_testinternalcapi' shared extension diff --git a/Include/internal/pycore_pymath.h b/Include/internal/pycore_pymath.h index eea8996ba68..532c5ceafb5 100644 --- a/Include/internal/pycore_pymath.h +++ b/Include/internal/pycore_pymath.h @@ -33,7 +33,7 @@ extern "C" { static inline void _Py_ADJUST_ERANGE1(double x) { if (errno == 0) { - if (x == Py_INFINITY || x == -Py_INFINITY) { + if (x == INFINITY || x == -INFINITY) { errno = ERANGE; } } @@ -44,8 +44,8 @@ static inline void _Py_ADJUST_ERANGE1(double x) static inline void _Py_ADJUST_ERANGE2(double x, double y) { - if (x == Py_INFINITY || x == -Py_INFINITY || - y == Py_INFINITY || y == -Py_INFINITY) + if (x == INFINITY || x == -INFINITY || + y == INFINITY || y == -INFINITY) { if (errno == 0) { errno = ERANGE; @@ -146,17 +146,17 @@ extern void _Py_set_387controlword(unsigned short); unsigned int old_fpcr, new_fpcr #define _Py_SET_53BIT_PRECISION_START \ do { \ - __asm__ ("fmove.l %%fpcr,%0" : "=g" (old_fpcr)); \ + __asm__ ("fmove.l %%fpcr,%0" : "=dm" (old_fpcr)); \ /* Set double precision / round to nearest. */ \ new_fpcr = (old_fpcr & ~0xf0) | 0x80; \ if (new_fpcr != old_fpcr) { \ - __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (new_fpcr));\ + __asm__ volatile ("fmove.l %0,%%fpcr" : : "dm" (new_fpcr)); \ } \ } while (0) #define _Py_SET_53BIT_PRECISION_END \ do { \ if (new_fpcr != old_fpcr) { \ - __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (old_fpcr)); \ + __asm__ volatile ("fmove.l %0,%%fpcr" : : "dm" (old_fpcr)); \ } \ } while (0) #endif @@ -182,8 +182,7 @@ extern void _Py_set_387controlword(unsigned short); // (extended precision), and we don't know how to change // the rounding precision. #if !defined(DOUBLE_IS_LITTLE_ENDIAN_IEEE754) && \ - !defined(DOUBLE_IS_BIG_ENDIAN_IEEE754) && \ - !defined(DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754) + !defined(DOUBLE_IS_BIG_ENDIAN_IEEE754) # define _PY_SHORT_FLOAT_REPR 0 #endif diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index f3f2ae0a140..05484e847f1 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -54,20 +54,43 @@ static inline int _PyMem_IsPtrFreed(const void *ptr) { uintptr_t value = (uintptr_t)ptr; #if SIZEOF_VOID_P == 8 - return (value == 0 + return (value <= 0xff // NULL, 0x1, 0x2, ..., 0xff || value == (uintptr_t)0xCDCDCDCDCDCDCDCD || value == (uintptr_t)0xDDDDDDDDDDDDDDDD - || value == (uintptr_t)0xFDFDFDFDFDFDFDFD); + || value == (uintptr_t)0xFDFDFDFDFDFDFDFD + || value >= (uintptr_t)0xFFFFFFFFFFFFFF00); // -0xff, ..., -2, -1 #elif SIZEOF_VOID_P == 4 - return (value == 0 + return (value <= 0xff || value == (uintptr_t)0xCDCDCDCD || value == (uintptr_t)0xDDDDDDDD - || value == (uintptr_t)0xFDFDFDFD); + || value == (uintptr_t)0xFDFDFDFD + || value >= (uintptr_t)0xFFFFFF00); #else # error "unknown pointer size" #endif } +// Similar to _PyMem_IsPtrFreed() but expects an 'unsigned long' instead of a +// pointer. +static inline int _PyMem_IsULongFreed(unsigned long value) +{ +#if SIZEOF_LONG == 8 + return (value == 0 + || value == (unsigned long)0xCDCDCDCDCDCDCDCD + || value == (unsigned long)0xDDDDDDDDDDDDDDDD + || value == (unsigned long)0xFDFDFDFDFDFDFDFD + || value == (unsigned long)0xFFFFFFFFFFFFFFFF); +#elif SIZEOF_LONG == 4 + return (value == 0 + || value == (unsigned long)0xCDCDCDCD + || value == (unsigned long)0xDDDDDDDD + || value == (unsigned long)0xFDFDFDFD + || value == (unsigned long)0xFFFFFFFF); +#else +# error "unknown long size" +#endif +} + extern int _PyMem_GetAllocatorName( const char *name, PyMemAllocatorName *allocator); diff --git a/Include/internal/pycore_pymem_init.h b/Include/internal/pycore_pymem_init.h index c593edc86d9..2a0e0817dcc 100644 --- a/Include/internal/pycore_pymem_init.h +++ b/Include/internal/pycore_pymem_init.h @@ -30,6 +30,12 @@ extern void* _PyMem_MiCalloc(void *, size_t, size_t); extern void _PyMem_MiFree(void *, void *); extern void* _PyMem_MiRealloc(void *, void *, size_t); # define PYMEM_ALLOC {NULL, _PyMem_MiMalloc, _PyMem_MiCalloc, _PyMem_MiRealloc, _PyMem_MiFree} +extern void* _PyMem_MiRawMalloc(void *, size_t); +extern void* _PyMem_MiRawCalloc(void *, size_t, size_t); +extern void _PyMem_MiRawFree(void *, void *); +extern void* _PyMem_MiRawRealloc(void *, void *, size_t); +# undef PYRAW_ALLOC +# define PYRAW_ALLOC {NULL, _PyMem_MiRawMalloc, _PyMem_MiRawCalloc, _PyMem_MiRawRealloc, _PyMem_MiRawFree} #elif defined(WITH_PYMALLOC) extern void* _PyObject_Malloc(void *, size_t); extern void* _PyObject_Calloc(void *, size_t, size_t); diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index ea3dfbd2eef..189a8dde9f0 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -89,8 +89,9 @@ _Py_ThreadCanHandleSignals(PyInterpreterState *interp) /* Variable and static inline functions for in-line access to current thread and interpreter state */ -#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) +#if !defined(Py_BUILD_CORE_MODULE) extern _Py_thread_local PyThreadState *_Py_tss_tstate; +extern _Py_thread_local PyInterpreterState *_Py_tss_interp; #endif #ifndef NDEBUG @@ -114,7 +115,7 @@ PyAPI_FUNC(PyThreadState *) _PyThreadState_GetCurrent(void); static inline PyThreadState* _PyThreadState_GET(void) { -#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) +#if !defined(Py_BUILD_CORE_MODULE) return _Py_tss_tstate; #else return _PyThreadState_GetCurrent(); @@ -204,11 +205,15 @@ _Py_EnsureFuncTstateNotNULL(const char *func, PyThreadState *tstate) See also PyInterpreterState_Get() and _PyGILState_GetInterpreterStateUnsafe(). */ static inline PyInterpreterState* _PyInterpreterState_GET(void) { - PyThreadState *tstate = _PyThreadState_GET(); #ifdef Py_DEBUG + PyThreadState *tstate = _PyThreadState_GET(); _Py_EnsureTstateNotNULL(tstate); #endif - return tstate->interp; +#if !defined(Py_BUILD_CORE_MODULE) + return _Py_tss_interp; +#else + return _PyThreadState_GET()->interp; +#endif } @@ -326,7 +331,11 @@ _Py_RecursionLimit_GetMargin(PyThreadState *tstate) _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; assert(_tstate->c_stack_hard_limit != 0); intptr_t here_addr = _Py_get_machine_stack_pointer(); +#if _Py_STACK_GROWS_DOWN return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, here_addr - (intptr_t)_tstate->c_stack_soft_limit, _PyOS_STACK_MARGIN_SHIFT); +#else + return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (intptr_t)_tstate->c_stack_soft_limit - here_addr, _PyOS_STACK_MARGIN_SHIFT); +#endif } #ifdef __cplusplus diff --git a/Include/internal/pycore_pystats.h b/Include/internal/pycore_pystats.h index f8af398a560..50ab21aa0f1 100644 --- a/Include/internal/pycore_pystats.h +++ b/Include/internal/pycore_pystats.h @@ -9,7 +9,7 @@ extern "C" { #endif #ifdef Py_STATS -extern void _Py_StatsOn(void); +extern int _Py_StatsOn(void); extern void _Py_StatsOff(void); extern void _Py_StatsClear(void); extern int _Py_PrintSpecializationStats(int to_file); diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index c2832098ddb..2a544edc431 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -33,14 +33,20 @@ extern const char* _Py_SourceAsString( PyCompilerFlags *cf, PyObject **cmd_copy); +extern PyObject * _Py_CompileStringObjectWithModule( + const char *str, + PyObject *filename, int start, + PyCompilerFlags *flags, int optimize, + PyObject *module); + /* Stack size, in "pointers". This must be large enough, so * no two calls to check recursion depth are more than this far * apart. In practice, that means it must be larger than the C * stack consumption of PyEval_EvalDefault */ -#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) -# define _PyOS_LOG2_STACK_MARGIN 12 -#elif defined(Py_DEBUG) && defined(WIN32) +#if (defined(Py_DEBUG) \ + || defined(_Py_ADDRESS_SANITIZER) \ + || defined(_Py_THREAD_SANITIZER)) # define _PyOS_LOG2_STACK_MARGIN 12 #else # define _PyOS_LOG2_STACK_MARGIN 11 @@ -54,6 +60,12 @@ extern const char* _Py_SourceAsString( # define _PyOS_STACK_MARGIN_SHIFT (_PyOS_LOG2_STACK_MARGIN + 2) #endif +#ifdef _Py_THREAD_SANITIZER +# define _PyOS_MIN_STACK_SIZE (_PyOS_STACK_MARGIN_BYTES * 6) +#else +# define _PyOS_MIN_STACK_SIZE (_PyOS_STACK_MARGIN_BYTES * 3) +#endif + #ifdef __cplusplus } diff --git a/Include/internal/pycore_qsbr.h b/Include/internal/pycore_qsbr.h index 1f9b3fcf777..eeca6fc472b 100644 --- a/Include/internal/pycore_qsbr.h +++ b/Include/internal/pycore_qsbr.h @@ -83,8 +83,9 @@ struct _qsbr_shared { // Minimum observed read sequence of all QSBR thread states uint64_t rd_seq; - // Array of QSBR thread states. + // Array of QSBR thread states (aligned to 64 bytes). struct _qsbr_pad *array; + void *array_raw; // raw allocation pointer (for free) Py_ssize_t size; // Freelist of unused _qsbr_thread_states (protected by mutex) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 7fc7f343fe6..fcd2ae9b1d1 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -56,6 +56,29 @@ _PyRuntimeState_SetFinalizing(_PyRuntimeState *runtime, PyThreadState *tstate) { } } +// Atomic so a thread that reads initialized=1 observes all writes +// from the initialization sequence (gh-146302). + +static inline int +_PyRuntimeState_GetCoreInitialized(_PyRuntimeState *runtime) { + return _Py_atomic_load_int(&runtime->core_initialized); +} + +static inline void +_PyRuntimeState_SetCoreInitialized(_PyRuntimeState *runtime, int initialized) { + _Py_atomic_store_int(&runtime->core_initialized, initialized); +} + +static inline int +_PyRuntimeState_GetInitialized(_PyRuntimeState *runtime) { + return _Py_atomic_load_int(&runtime->initialized); +} + +static inline void +_PyRuntimeState_SetInitialized(_PyRuntimeState *runtime, int initialized) { + _Py_atomic_store_int(&runtime->initialized, initialized); +} + #ifdef __cplusplus } diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index b182f7825a2..6c48ac0dccf 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -13,7 +13,7 @@ extern "C" { #include "pycore_debug_offsets.h" // _Py_DebugOffsets_INIT() #include "pycore_dtoa.h" // _dtoa_state_INIT() #include "pycore_faulthandler.h" // _faulthandler_runtime_state_INIT -#include "pycore_floatobject.h" // _py_float_format_unknown +#include "pycore_floatobject.h" // _py_float_format_* #include "pycore_function.h" #include "pycore_hamt.h" // _PyHamt_BitmapNode_Type #include "pycore_import.h" // IMPORTS_INIT @@ -84,10 +84,6 @@ extern PyTypeObject _PyExc_MemoryError; .stoptheworld = { \ .is_global = 1, \ }, \ - .float_state = { \ - .float_format = _py_float_format_unknown, \ - .double_format = _py_float_format_unknown, \ - }, \ .types = { \ .next_version_tag = _Py_TYPE_VERSION_NEXT, \ }, \ @@ -134,13 +130,7 @@ extern PyTypeObject _PyExc_MemoryError; }, \ .gc = { \ .enabled = 1, \ - .young = { .threshold = 2000, }, \ - .old = { \ - { .threshold = 10, }, \ - { .threshold = 0, }, \ - }, \ - .work_to_do = -5000, \ - .phase = GC_PHASE_MARK, \ + GC_GENERATION_INIT \ }, \ .qsbr = { \ .wr_seq = QSBR_INITIAL, \ @@ -233,4 +223,4 @@ extern PyTypeObject _PyExc_MemoryError; #ifdef __cplusplus } #endif -#endif /* !Py_INTERNAL_RUNTIME_INIT_H */ \ No newline at end of file +#endif /* !Py_INTERNAL_RUNTIME_INIT_H */ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 3ce7200ffeb..892c3cdd962 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -275,6 +275,774 @@ extern "C" { _PyLong_DIGIT_INIT(254), \ _PyLong_DIGIT_INIT(255), \ _PyLong_DIGIT_INIT(256), \ + _PyLong_DIGIT_INIT(257), \ + _PyLong_DIGIT_INIT(258), \ + _PyLong_DIGIT_INIT(259), \ + _PyLong_DIGIT_INIT(260), \ + _PyLong_DIGIT_INIT(261), \ + _PyLong_DIGIT_INIT(262), \ + _PyLong_DIGIT_INIT(263), \ + _PyLong_DIGIT_INIT(264), \ + _PyLong_DIGIT_INIT(265), \ + _PyLong_DIGIT_INIT(266), \ + _PyLong_DIGIT_INIT(267), \ + _PyLong_DIGIT_INIT(268), \ + _PyLong_DIGIT_INIT(269), \ + _PyLong_DIGIT_INIT(270), \ + _PyLong_DIGIT_INIT(271), \ + _PyLong_DIGIT_INIT(272), \ + _PyLong_DIGIT_INIT(273), \ + _PyLong_DIGIT_INIT(274), \ + _PyLong_DIGIT_INIT(275), \ + _PyLong_DIGIT_INIT(276), \ + _PyLong_DIGIT_INIT(277), \ + _PyLong_DIGIT_INIT(278), \ + _PyLong_DIGIT_INIT(279), \ + _PyLong_DIGIT_INIT(280), \ + _PyLong_DIGIT_INIT(281), \ + _PyLong_DIGIT_INIT(282), \ + _PyLong_DIGIT_INIT(283), \ + _PyLong_DIGIT_INIT(284), \ + _PyLong_DIGIT_INIT(285), \ + _PyLong_DIGIT_INIT(286), \ + _PyLong_DIGIT_INIT(287), \ + _PyLong_DIGIT_INIT(288), \ + _PyLong_DIGIT_INIT(289), \ + _PyLong_DIGIT_INIT(290), \ + _PyLong_DIGIT_INIT(291), \ + _PyLong_DIGIT_INIT(292), \ + _PyLong_DIGIT_INIT(293), \ + _PyLong_DIGIT_INIT(294), \ + _PyLong_DIGIT_INIT(295), \ + _PyLong_DIGIT_INIT(296), \ + _PyLong_DIGIT_INIT(297), \ + _PyLong_DIGIT_INIT(298), \ + _PyLong_DIGIT_INIT(299), \ + _PyLong_DIGIT_INIT(300), \ + _PyLong_DIGIT_INIT(301), \ + _PyLong_DIGIT_INIT(302), \ + _PyLong_DIGIT_INIT(303), \ + _PyLong_DIGIT_INIT(304), \ + _PyLong_DIGIT_INIT(305), \ + _PyLong_DIGIT_INIT(306), \ + _PyLong_DIGIT_INIT(307), \ + _PyLong_DIGIT_INIT(308), \ + _PyLong_DIGIT_INIT(309), \ + _PyLong_DIGIT_INIT(310), \ + _PyLong_DIGIT_INIT(311), \ + _PyLong_DIGIT_INIT(312), \ + _PyLong_DIGIT_INIT(313), \ + _PyLong_DIGIT_INIT(314), \ + _PyLong_DIGIT_INIT(315), \ + _PyLong_DIGIT_INIT(316), \ + _PyLong_DIGIT_INIT(317), \ + _PyLong_DIGIT_INIT(318), \ + _PyLong_DIGIT_INIT(319), \ + _PyLong_DIGIT_INIT(320), \ + _PyLong_DIGIT_INIT(321), \ + _PyLong_DIGIT_INIT(322), \ + _PyLong_DIGIT_INIT(323), \ + _PyLong_DIGIT_INIT(324), \ + _PyLong_DIGIT_INIT(325), \ + _PyLong_DIGIT_INIT(326), \ + _PyLong_DIGIT_INIT(327), \ + _PyLong_DIGIT_INIT(328), \ + _PyLong_DIGIT_INIT(329), \ + _PyLong_DIGIT_INIT(330), \ + _PyLong_DIGIT_INIT(331), \ + _PyLong_DIGIT_INIT(332), \ + _PyLong_DIGIT_INIT(333), \ + _PyLong_DIGIT_INIT(334), \ + _PyLong_DIGIT_INIT(335), \ + _PyLong_DIGIT_INIT(336), \ + _PyLong_DIGIT_INIT(337), \ + _PyLong_DIGIT_INIT(338), \ + _PyLong_DIGIT_INIT(339), \ + _PyLong_DIGIT_INIT(340), \ + _PyLong_DIGIT_INIT(341), \ + _PyLong_DIGIT_INIT(342), \ + _PyLong_DIGIT_INIT(343), \ + _PyLong_DIGIT_INIT(344), \ + _PyLong_DIGIT_INIT(345), \ + _PyLong_DIGIT_INIT(346), \ + _PyLong_DIGIT_INIT(347), \ + _PyLong_DIGIT_INIT(348), \ + _PyLong_DIGIT_INIT(349), \ + _PyLong_DIGIT_INIT(350), \ + _PyLong_DIGIT_INIT(351), \ + _PyLong_DIGIT_INIT(352), \ + _PyLong_DIGIT_INIT(353), \ + _PyLong_DIGIT_INIT(354), \ + _PyLong_DIGIT_INIT(355), \ + _PyLong_DIGIT_INIT(356), \ + _PyLong_DIGIT_INIT(357), \ + _PyLong_DIGIT_INIT(358), \ + _PyLong_DIGIT_INIT(359), \ + _PyLong_DIGIT_INIT(360), \ + _PyLong_DIGIT_INIT(361), \ + _PyLong_DIGIT_INIT(362), \ + _PyLong_DIGIT_INIT(363), \ + _PyLong_DIGIT_INIT(364), \ + _PyLong_DIGIT_INIT(365), \ + _PyLong_DIGIT_INIT(366), \ + _PyLong_DIGIT_INIT(367), \ + _PyLong_DIGIT_INIT(368), \ + _PyLong_DIGIT_INIT(369), \ + _PyLong_DIGIT_INIT(370), \ + _PyLong_DIGIT_INIT(371), \ + _PyLong_DIGIT_INIT(372), \ + _PyLong_DIGIT_INIT(373), \ + _PyLong_DIGIT_INIT(374), \ + _PyLong_DIGIT_INIT(375), \ + _PyLong_DIGIT_INIT(376), \ + _PyLong_DIGIT_INIT(377), \ + _PyLong_DIGIT_INIT(378), \ + _PyLong_DIGIT_INIT(379), \ + _PyLong_DIGIT_INIT(380), \ + _PyLong_DIGIT_INIT(381), \ + _PyLong_DIGIT_INIT(382), \ + _PyLong_DIGIT_INIT(383), \ + _PyLong_DIGIT_INIT(384), \ + _PyLong_DIGIT_INIT(385), \ + _PyLong_DIGIT_INIT(386), \ + _PyLong_DIGIT_INIT(387), \ + _PyLong_DIGIT_INIT(388), \ + _PyLong_DIGIT_INIT(389), \ + _PyLong_DIGIT_INIT(390), \ + _PyLong_DIGIT_INIT(391), \ + _PyLong_DIGIT_INIT(392), \ + _PyLong_DIGIT_INIT(393), \ + _PyLong_DIGIT_INIT(394), \ + _PyLong_DIGIT_INIT(395), \ + _PyLong_DIGIT_INIT(396), \ + _PyLong_DIGIT_INIT(397), \ + _PyLong_DIGIT_INIT(398), \ + _PyLong_DIGIT_INIT(399), \ + _PyLong_DIGIT_INIT(400), \ + _PyLong_DIGIT_INIT(401), \ + _PyLong_DIGIT_INIT(402), \ + _PyLong_DIGIT_INIT(403), \ + _PyLong_DIGIT_INIT(404), \ + _PyLong_DIGIT_INIT(405), \ + _PyLong_DIGIT_INIT(406), \ + _PyLong_DIGIT_INIT(407), \ + _PyLong_DIGIT_INIT(408), \ + _PyLong_DIGIT_INIT(409), \ + _PyLong_DIGIT_INIT(410), \ + _PyLong_DIGIT_INIT(411), \ + _PyLong_DIGIT_INIT(412), \ + _PyLong_DIGIT_INIT(413), \ + _PyLong_DIGIT_INIT(414), \ + _PyLong_DIGIT_INIT(415), \ + _PyLong_DIGIT_INIT(416), \ + _PyLong_DIGIT_INIT(417), \ + _PyLong_DIGIT_INIT(418), \ + _PyLong_DIGIT_INIT(419), \ + _PyLong_DIGIT_INIT(420), \ + _PyLong_DIGIT_INIT(421), \ + _PyLong_DIGIT_INIT(422), \ + _PyLong_DIGIT_INIT(423), \ + _PyLong_DIGIT_INIT(424), \ + _PyLong_DIGIT_INIT(425), \ + _PyLong_DIGIT_INIT(426), \ + _PyLong_DIGIT_INIT(427), \ + _PyLong_DIGIT_INIT(428), \ + _PyLong_DIGIT_INIT(429), \ + _PyLong_DIGIT_INIT(430), \ + _PyLong_DIGIT_INIT(431), \ + _PyLong_DIGIT_INIT(432), \ + _PyLong_DIGIT_INIT(433), \ + _PyLong_DIGIT_INIT(434), \ + _PyLong_DIGIT_INIT(435), \ + _PyLong_DIGIT_INIT(436), \ + _PyLong_DIGIT_INIT(437), \ + _PyLong_DIGIT_INIT(438), \ + _PyLong_DIGIT_INIT(439), \ + _PyLong_DIGIT_INIT(440), \ + _PyLong_DIGIT_INIT(441), \ + _PyLong_DIGIT_INIT(442), \ + _PyLong_DIGIT_INIT(443), \ + _PyLong_DIGIT_INIT(444), \ + _PyLong_DIGIT_INIT(445), \ + _PyLong_DIGIT_INIT(446), \ + _PyLong_DIGIT_INIT(447), \ + _PyLong_DIGIT_INIT(448), \ + _PyLong_DIGIT_INIT(449), \ + _PyLong_DIGIT_INIT(450), \ + _PyLong_DIGIT_INIT(451), \ + _PyLong_DIGIT_INIT(452), \ + _PyLong_DIGIT_INIT(453), \ + _PyLong_DIGIT_INIT(454), \ + _PyLong_DIGIT_INIT(455), \ + _PyLong_DIGIT_INIT(456), \ + _PyLong_DIGIT_INIT(457), \ + _PyLong_DIGIT_INIT(458), \ + _PyLong_DIGIT_INIT(459), \ + _PyLong_DIGIT_INIT(460), \ + _PyLong_DIGIT_INIT(461), \ + _PyLong_DIGIT_INIT(462), \ + _PyLong_DIGIT_INIT(463), \ + _PyLong_DIGIT_INIT(464), \ + _PyLong_DIGIT_INIT(465), \ + _PyLong_DIGIT_INIT(466), \ + _PyLong_DIGIT_INIT(467), \ + _PyLong_DIGIT_INIT(468), \ + _PyLong_DIGIT_INIT(469), \ + _PyLong_DIGIT_INIT(470), \ + _PyLong_DIGIT_INIT(471), \ + _PyLong_DIGIT_INIT(472), \ + _PyLong_DIGIT_INIT(473), \ + _PyLong_DIGIT_INIT(474), \ + _PyLong_DIGIT_INIT(475), \ + _PyLong_DIGIT_INIT(476), \ + _PyLong_DIGIT_INIT(477), \ + _PyLong_DIGIT_INIT(478), \ + _PyLong_DIGIT_INIT(479), \ + _PyLong_DIGIT_INIT(480), \ + _PyLong_DIGIT_INIT(481), \ + _PyLong_DIGIT_INIT(482), \ + _PyLong_DIGIT_INIT(483), \ + _PyLong_DIGIT_INIT(484), \ + _PyLong_DIGIT_INIT(485), \ + _PyLong_DIGIT_INIT(486), \ + _PyLong_DIGIT_INIT(487), \ + _PyLong_DIGIT_INIT(488), \ + _PyLong_DIGIT_INIT(489), \ + _PyLong_DIGIT_INIT(490), \ + _PyLong_DIGIT_INIT(491), \ + _PyLong_DIGIT_INIT(492), \ + _PyLong_DIGIT_INIT(493), \ + _PyLong_DIGIT_INIT(494), \ + _PyLong_DIGIT_INIT(495), \ + _PyLong_DIGIT_INIT(496), \ + _PyLong_DIGIT_INIT(497), \ + _PyLong_DIGIT_INIT(498), \ + _PyLong_DIGIT_INIT(499), \ + _PyLong_DIGIT_INIT(500), \ + _PyLong_DIGIT_INIT(501), \ + _PyLong_DIGIT_INIT(502), \ + _PyLong_DIGIT_INIT(503), \ + _PyLong_DIGIT_INIT(504), \ + _PyLong_DIGIT_INIT(505), \ + _PyLong_DIGIT_INIT(506), \ + _PyLong_DIGIT_INIT(507), \ + _PyLong_DIGIT_INIT(508), \ + _PyLong_DIGIT_INIT(509), \ + _PyLong_DIGIT_INIT(510), \ + _PyLong_DIGIT_INIT(511), \ + _PyLong_DIGIT_INIT(512), \ + _PyLong_DIGIT_INIT(513), \ + _PyLong_DIGIT_INIT(514), \ + _PyLong_DIGIT_INIT(515), \ + _PyLong_DIGIT_INIT(516), \ + _PyLong_DIGIT_INIT(517), \ + _PyLong_DIGIT_INIT(518), \ + _PyLong_DIGIT_INIT(519), \ + _PyLong_DIGIT_INIT(520), \ + _PyLong_DIGIT_INIT(521), \ + _PyLong_DIGIT_INIT(522), \ + _PyLong_DIGIT_INIT(523), \ + _PyLong_DIGIT_INIT(524), \ + _PyLong_DIGIT_INIT(525), \ + _PyLong_DIGIT_INIT(526), \ + _PyLong_DIGIT_INIT(527), \ + _PyLong_DIGIT_INIT(528), \ + _PyLong_DIGIT_INIT(529), \ + _PyLong_DIGIT_INIT(530), \ + _PyLong_DIGIT_INIT(531), \ + _PyLong_DIGIT_INIT(532), \ + _PyLong_DIGIT_INIT(533), \ + _PyLong_DIGIT_INIT(534), \ + _PyLong_DIGIT_INIT(535), \ + _PyLong_DIGIT_INIT(536), \ + _PyLong_DIGIT_INIT(537), \ + _PyLong_DIGIT_INIT(538), \ + _PyLong_DIGIT_INIT(539), \ + _PyLong_DIGIT_INIT(540), \ + _PyLong_DIGIT_INIT(541), \ + _PyLong_DIGIT_INIT(542), \ + _PyLong_DIGIT_INIT(543), \ + _PyLong_DIGIT_INIT(544), \ + _PyLong_DIGIT_INIT(545), \ + _PyLong_DIGIT_INIT(546), \ + _PyLong_DIGIT_INIT(547), \ + _PyLong_DIGIT_INIT(548), \ + _PyLong_DIGIT_INIT(549), \ + _PyLong_DIGIT_INIT(550), \ + _PyLong_DIGIT_INIT(551), \ + _PyLong_DIGIT_INIT(552), \ + _PyLong_DIGIT_INIT(553), \ + _PyLong_DIGIT_INIT(554), \ + _PyLong_DIGIT_INIT(555), \ + _PyLong_DIGIT_INIT(556), \ + _PyLong_DIGIT_INIT(557), \ + _PyLong_DIGIT_INIT(558), \ + _PyLong_DIGIT_INIT(559), \ + _PyLong_DIGIT_INIT(560), \ + _PyLong_DIGIT_INIT(561), \ + _PyLong_DIGIT_INIT(562), \ + _PyLong_DIGIT_INIT(563), \ + _PyLong_DIGIT_INIT(564), \ + _PyLong_DIGIT_INIT(565), \ + _PyLong_DIGIT_INIT(566), \ + _PyLong_DIGIT_INIT(567), \ + _PyLong_DIGIT_INIT(568), \ + _PyLong_DIGIT_INIT(569), \ + _PyLong_DIGIT_INIT(570), \ + _PyLong_DIGIT_INIT(571), \ + _PyLong_DIGIT_INIT(572), \ + _PyLong_DIGIT_INIT(573), \ + _PyLong_DIGIT_INIT(574), \ + _PyLong_DIGIT_INIT(575), \ + _PyLong_DIGIT_INIT(576), \ + _PyLong_DIGIT_INIT(577), \ + _PyLong_DIGIT_INIT(578), \ + _PyLong_DIGIT_INIT(579), \ + _PyLong_DIGIT_INIT(580), \ + _PyLong_DIGIT_INIT(581), \ + _PyLong_DIGIT_INIT(582), \ + _PyLong_DIGIT_INIT(583), \ + _PyLong_DIGIT_INIT(584), \ + _PyLong_DIGIT_INIT(585), \ + _PyLong_DIGIT_INIT(586), \ + _PyLong_DIGIT_INIT(587), \ + _PyLong_DIGIT_INIT(588), \ + _PyLong_DIGIT_INIT(589), \ + _PyLong_DIGIT_INIT(590), \ + _PyLong_DIGIT_INIT(591), \ + _PyLong_DIGIT_INIT(592), \ + _PyLong_DIGIT_INIT(593), \ + _PyLong_DIGIT_INIT(594), \ + _PyLong_DIGIT_INIT(595), \ + _PyLong_DIGIT_INIT(596), \ + _PyLong_DIGIT_INIT(597), \ + _PyLong_DIGIT_INIT(598), \ + _PyLong_DIGIT_INIT(599), \ + _PyLong_DIGIT_INIT(600), \ + _PyLong_DIGIT_INIT(601), \ + _PyLong_DIGIT_INIT(602), \ + _PyLong_DIGIT_INIT(603), \ + _PyLong_DIGIT_INIT(604), \ + _PyLong_DIGIT_INIT(605), \ + _PyLong_DIGIT_INIT(606), \ + _PyLong_DIGIT_INIT(607), \ + _PyLong_DIGIT_INIT(608), \ + _PyLong_DIGIT_INIT(609), \ + _PyLong_DIGIT_INIT(610), \ + _PyLong_DIGIT_INIT(611), \ + _PyLong_DIGIT_INIT(612), \ + _PyLong_DIGIT_INIT(613), \ + _PyLong_DIGIT_INIT(614), \ + _PyLong_DIGIT_INIT(615), \ + _PyLong_DIGIT_INIT(616), \ + _PyLong_DIGIT_INIT(617), \ + _PyLong_DIGIT_INIT(618), \ + _PyLong_DIGIT_INIT(619), \ + _PyLong_DIGIT_INIT(620), \ + _PyLong_DIGIT_INIT(621), \ + _PyLong_DIGIT_INIT(622), \ + _PyLong_DIGIT_INIT(623), \ + _PyLong_DIGIT_INIT(624), \ + _PyLong_DIGIT_INIT(625), \ + _PyLong_DIGIT_INIT(626), \ + _PyLong_DIGIT_INIT(627), \ + _PyLong_DIGIT_INIT(628), \ + _PyLong_DIGIT_INIT(629), \ + _PyLong_DIGIT_INIT(630), \ + _PyLong_DIGIT_INIT(631), \ + _PyLong_DIGIT_INIT(632), \ + _PyLong_DIGIT_INIT(633), \ + _PyLong_DIGIT_INIT(634), \ + _PyLong_DIGIT_INIT(635), \ + _PyLong_DIGIT_INIT(636), \ + _PyLong_DIGIT_INIT(637), \ + _PyLong_DIGIT_INIT(638), \ + _PyLong_DIGIT_INIT(639), \ + _PyLong_DIGIT_INIT(640), \ + _PyLong_DIGIT_INIT(641), \ + _PyLong_DIGIT_INIT(642), \ + _PyLong_DIGIT_INIT(643), \ + _PyLong_DIGIT_INIT(644), \ + _PyLong_DIGIT_INIT(645), \ + _PyLong_DIGIT_INIT(646), \ + _PyLong_DIGIT_INIT(647), \ + _PyLong_DIGIT_INIT(648), \ + _PyLong_DIGIT_INIT(649), \ + _PyLong_DIGIT_INIT(650), \ + _PyLong_DIGIT_INIT(651), \ + _PyLong_DIGIT_INIT(652), \ + _PyLong_DIGIT_INIT(653), \ + _PyLong_DIGIT_INIT(654), \ + _PyLong_DIGIT_INIT(655), \ + _PyLong_DIGIT_INIT(656), \ + _PyLong_DIGIT_INIT(657), \ + _PyLong_DIGIT_INIT(658), \ + _PyLong_DIGIT_INIT(659), \ + _PyLong_DIGIT_INIT(660), \ + _PyLong_DIGIT_INIT(661), \ + _PyLong_DIGIT_INIT(662), \ + _PyLong_DIGIT_INIT(663), \ + _PyLong_DIGIT_INIT(664), \ + _PyLong_DIGIT_INIT(665), \ + _PyLong_DIGIT_INIT(666), \ + _PyLong_DIGIT_INIT(667), \ + _PyLong_DIGIT_INIT(668), \ + _PyLong_DIGIT_INIT(669), \ + _PyLong_DIGIT_INIT(670), \ + _PyLong_DIGIT_INIT(671), \ + _PyLong_DIGIT_INIT(672), \ + _PyLong_DIGIT_INIT(673), \ + _PyLong_DIGIT_INIT(674), \ + _PyLong_DIGIT_INIT(675), \ + _PyLong_DIGIT_INIT(676), \ + _PyLong_DIGIT_INIT(677), \ + _PyLong_DIGIT_INIT(678), \ + _PyLong_DIGIT_INIT(679), \ + _PyLong_DIGIT_INIT(680), \ + _PyLong_DIGIT_INIT(681), \ + _PyLong_DIGIT_INIT(682), \ + _PyLong_DIGIT_INIT(683), \ + _PyLong_DIGIT_INIT(684), \ + _PyLong_DIGIT_INIT(685), \ + _PyLong_DIGIT_INIT(686), \ + _PyLong_DIGIT_INIT(687), \ + _PyLong_DIGIT_INIT(688), \ + _PyLong_DIGIT_INIT(689), \ + _PyLong_DIGIT_INIT(690), \ + _PyLong_DIGIT_INIT(691), \ + _PyLong_DIGIT_INIT(692), \ + _PyLong_DIGIT_INIT(693), \ + _PyLong_DIGIT_INIT(694), \ + _PyLong_DIGIT_INIT(695), \ + _PyLong_DIGIT_INIT(696), \ + _PyLong_DIGIT_INIT(697), \ + _PyLong_DIGIT_INIT(698), \ + _PyLong_DIGIT_INIT(699), \ + _PyLong_DIGIT_INIT(700), \ + _PyLong_DIGIT_INIT(701), \ + _PyLong_DIGIT_INIT(702), \ + _PyLong_DIGIT_INIT(703), \ + _PyLong_DIGIT_INIT(704), \ + _PyLong_DIGIT_INIT(705), \ + _PyLong_DIGIT_INIT(706), \ + _PyLong_DIGIT_INIT(707), \ + _PyLong_DIGIT_INIT(708), \ + _PyLong_DIGIT_INIT(709), \ + _PyLong_DIGIT_INIT(710), \ + _PyLong_DIGIT_INIT(711), \ + _PyLong_DIGIT_INIT(712), \ + _PyLong_DIGIT_INIT(713), \ + _PyLong_DIGIT_INIT(714), \ + _PyLong_DIGIT_INIT(715), \ + _PyLong_DIGIT_INIT(716), \ + _PyLong_DIGIT_INIT(717), \ + _PyLong_DIGIT_INIT(718), \ + _PyLong_DIGIT_INIT(719), \ + _PyLong_DIGIT_INIT(720), \ + _PyLong_DIGIT_INIT(721), \ + _PyLong_DIGIT_INIT(722), \ + _PyLong_DIGIT_INIT(723), \ + _PyLong_DIGIT_INIT(724), \ + _PyLong_DIGIT_INIT(725), \ + _PyLong_DIGIT_INIT(726), \ + _PyLong_DIGIT_INIT(727), \ + _PyLong_DIGIT_INIT(728), \ + _PyLong_DIGIT_INIT(729), \ + _PyLong_DIGIT_INIT(730), \ + _PyLong_DIGIT_INIT(731), \ + _PyLong_DIGIT_INIT(732), \ + _PyLong_DIGIT_INIT(733), \ + _PyLong_DIGIT_INIT(734), \ + _PyLong_DIGIT_INIT(735), \ + _PyLong_DIGIT_INIT(736), \ + _PyLong_DIGIT_INIT(737), \ + _PyLong_DIGIT_INIT(738), \ + _PyLong_DIGIT_INIT(739), \ + _PyLong_DIGIT_INIT(740), \ + _PyLong_DIGIT_INIT(741), \ + _PyLong_DIGIT_INIT(742), \ + _PyLong_DIGIT_INIT(743), \ + _PyLong_DIGIT_INIT(744), \ + _PyLong_DIGIT_INIT(745), \ + _PyLong_DIGIT_INIT(746), \ + _PyLong_DIGIT_INIT(747), \ + _PyLong_DIGIT_INIT(748), \ + _PyLong_DIGIT_INIT(749), \ + _PyLong_DIGIT_INIT(750), \ + _PyLong_DIGIT_INIT(751), \ + _PyLong_DIGIT_INIT(752), \ + _PyLong_DIGIT_INIT(753), \ + _PyLong_DIGIT_INIT(754), \ + _PyLong_DIGIT_INIT(755), \ + _PyLong_DIGIT_INIT(756), \ + _PyLong_DIGIT_INIT(757), \ + _PyLong_DIGIT_INIT(758), \ + _PyLong_DIGIT_INIT(759), \ + _PyLong_DIGIT_INIT(760), \ + _PyLong_DIGIT_INIT(761), \ + _PyLong_DIGIT_INIT(762), \ + _PyLong_DIGIT_INIT(763), \ + _PyLong_DIGIT_INIT(764), \ + _PyLong_DIGIT_INIT(765), \ + _PyLong_DIGIT_INIT(766), \ + _PyLong_DIGIT_INIT(767), \ + _PyLong_DIGIT_INIT(768), \ + _PyLong_DIGIT_INIT(769), \ + _PyLong_DIGIT_INIT(770), \ + _PyLong_DIGIT_INIT(771), \ + _PyLong_DIGIT_INIT(772), \ + _PyLong_DIGIT_INIT(773), \ + _PyLong_DIGIT_INIT(774), \ + _PyLong_DIGIT_INIT(775), \ + _PyLong_DIGIT_INIT(776), \ + _PyLong_DIGIT_INIT(777), \ + _PyLong_DIGIT_INIT(778), \ + _PyLong_DIGIT_INIT(779), \ + _PyLong_DIGIT_INIT(780), \ + _PyLong_DIGIT_INIT(781), \ + _PyLong_DIGIT_INIT(782), \ + _PyLong_DIGIT_INIT(783), \ + _PyLong_DIGIT_INIT(784), \ + _PyLong_DIGIT_INIT(785), \ + _PyLong_DIGIT_INIT(786), \ + _PyLong_DIGIT_INIT(787), \ + _PyLong_DIGIT_INIT(788), \ + _PyLong_DIGIT_INIT(789), \ + _PyLong_DIGIT_INIT(790), \ + _PyLong_DIGIT_INIT(791), \ + _PyLong_DIGIT_INIT(792), \ + _PyLong_DIGIT_INIT(793), \ + _PyLong_DIGIT_INIT(794), \ + _PyLong_DIGIT_INIT(795), \ + _PyLong_DIGIT_INIT(796), \ + _PyLong_DIGIT_INIT(797), \ + _PyLong_DIGIT_INIT(798), \ + _PyLong_DIGIT_INIT(799), \ + _PyLong_DIGIT_INIT(800), \ + _PyLong_DIGIT_INIT(801), \ + _PyLong_DIGIT_INIT(802), \ + _PyLong_DIGIT_INIT(803), \ + _PyLong_DIGIT_INIT(804), \ + _PyLong_DIGIT_INIT(805), \ + _PyLong_DIGIT_INIT(806), \ + _PyLong_DIGIT_INIT(807), \ + _PyLong_DIGIT_INIT(808), \ + _PyLong_DIGIT_INIT(809), \ + _PyLong_DIGIT_INIT(810), \ + _PyLong_DIGIT_INIT(811), \ + _PyLong_DIGIT_INIT(812), \ + _PyLong_DIGIT_INIT(813), \ + _PyLong_DIGIT_INIT(814), \ + _PyLong_DIGIT_INIT(815), \ + _PyLong_DIGIT_INIT(816), \ + _PyLong_DIGIT_INIT(817), \ + _PyLong_DIGIT_INIT(818), \ + _PyLong_DIGIT_INIT(819), \ + _PyLong_DIGIT_INIT(820), \ + _PyLong_DIGIT_INIT(821), \ + _PyLong_DIGIT_INIT(822), \ + _PyLong_DIGIT_INIT(823), \ + _PyLong_DIGIT_INIT(824), \ + _PyLong_DIGIT_INIT(825), \ + _PyLong_DIGIT_INIT(826), \ + _PyLong_DIGIT_INIT(827), \ + _PyLong_DIGIT_INIT(828), \ + _PyLong_DIGIT_INIT(829), \ + _PyLong_DIGIT_INIT(830), \ + _PyLong_DIGIT_INIT(831), \ + _PyLong_DIGIT_INIT(832), \ + _PyLong_DIGIT_INIT(833), \ + _PyLong_DIGIT_INIT(834), \ + _PyLong_DIGIT_INIT(835), \ + _PyLong_DIGIT_INIT(836), \ + _PyLong_DIGIT_INIT(837), \ + _PyLong_DIGIT_INIT(838), \ + _PyLong_DIGIT_INIT(839), \ + _PyLong_DIGIT_INIT(840), \ + _PyLong_DIGIT_INIT(841), \ + _PyLong_DIGIT_INIT(842), \ + _PyLong_DIGIT_INIT(843), \ + _PyLong_DIGIT_INIT(844), \ + _PyLong_DIGIT_INIT(845), \ + _PyLong_DIGIT_INIT(846), \ + _PyLong_DIGIT_INIT(847), \ + _PyLong_DIGIT_INIT(848), \ + _PyLong_DIGIT_INIT(849), \ + _PyLong_DIGIT_INIT(850), \ + _PyLong_DIGIT_INIT(851), \ + _PyLong_DIGIT_INIT(852), \ + _PyLong_DIGIT_INIT(853), \ + _PyLong_DIGIT_INIT(854), \ + _PyLong_DIGIT_INIT(855), \ + _PyLong_DIGIT_INIT(856), \ + _PyLong_DIGIT_INIT(857), \ + _PyLong_DIGIT_INIT(858), \ + _PyLong_DIGIT_INIT(859), \ + _PyLong_DIGIT_INIT(860), \ + _PyLong_DIGIT_INIT(861), \ + _PyLong_DIGIT_INIT(862), \ + _PyLong_DIGIT_INIT(863), \ + _PyLong_DIGIT_INIT(864), \ + _PyLong_DIGIT_INIT(865), \ + _PyLong_DIGIT_INIT(866), \ + _PyLong_DIGIT_INIT(867), \ + _PyLong_DIGIT_INIT(868), \ + _PyLong_DIGIT_INIT(869), \ + _PyLong_DIGIT_INIT(870), \ + _PyLong_DIGIT_INIT(871), \ + _PyLong_DIGIT_INIT(872), \ + _PyLong_DIGIT_INIT(873), \ + _PyLong_DIGIT_INIT(874), \ + _PyLong_DIGIT_INIT(875), \ + _PyLong_DIGIT_INIT(876), \ + _PyLong_DIGIT_INIT(877), \ + _PyLong_DIGIT_INIT(878), \ + _PyLong_DIGIT_INIT(879), \ + _PyLong_DIGIT_INIT(880), \ + _PyLong_DIGIT_INIT(881), \ + _PyLong_DIGIT_INIT(882), \ + _PyLong_DIGIT_INIT(883), \ + _PyLong_DIGIT_INIT(884), \ + _PyLong_DIGIT_INIT(885), \ + _PyLong_DIGIT_INIT(886), \ + _PyLong_DIGIT_INIT(887), \ + _PyLong_DIGIT_INIT(888), \ + _PyLong_DIGIT_INIT(889), \ + _PyLong_DIGIT_INIT(890), \ + _PyLong_DIGIT_INIT(891), \ + _PyLong_DIGIT_INIT(892), \ + _PyLong_DIGIT_INIT(893), \ + _PyLong_DIGIT_INIT(894), \ + _PyLong_DIGIT_INIT(895), \ + _PyLong_DIGIT_INIT(896), \ + _PyLong_DIGIT_INIT(897), \ + _PyLong_DIGIT_INIT(898), \ + _PyLong_DIGIT_INIT(899), \ + _PyLong_DIGIT_INIT(900), \ + _PyLong_DIGIT_INIT(901), \ + _PyLong_DIGIT_INIT(902), \ + _PyLong_DIGIT_INIT(903), \ + _PyLong_DIGIT_INIT(904), \ + _PyLong_DIGIT_INIT(905), \ + _PyLong_DIGIT_INIT(906), \ + _PyLong_DIGIT_INIT(907), \ + _PyLong_DIGIT_INIT(908), \ + _PyLong_DIGIT_INIT(909), \ + _PyLong_DIGIT_INIT(910), \ + _PyLong_DIGIT_INIT(911), \ + _PyLong_DIGIT_INIT(912), \ + _PyLong_DIGIT_INIT(913), \ + _PyLong_DIGIT_INIT(914), \ + _PyLong_DIGIT_INIT(915), \ + _PyLong_DIGIT_INIT(916), \ + _PyLong_DIGIT_INIT(917), \ + _PyLong_DIGIT_INIT(918), \ + _PyLong_DIGIT_INIT(919), \ + _PyLong_DIGIT_INIT(920), \ + _PyLong_DIGIT_INIT(921), \ + _PyLong_DIGIT_INIT(922), \ + _PyLong_DIGIT_INIT(923), \ + _PyLong_DIGIT_INIT(924), \ + _PyLong_DIGIT_INIT(925), \ + _PyLong_DIGIT_INIT(926), \ + _PyLong_DIGIT_INIT(927), \ + _PyLong_DIGIT_INIT(928), \ + _PyLong_DIGIT_INIT(929), \ + _PyLong_DIGIT_INIT(930), \ + _PyLong_DIGIT_INIT(931), \ + _PyLong_DIGIT_INIT(932), \ + _PyLong_DIGIT_INIT(933), \ + _PyLong_DIGIT_INIT(934), \ + _PyLong_DIGIT_INIT(935), \ + _PyLong_DIGIT_INIT(936), \ + _PyLong_DIGIT_INIT(937), \ + _PyLong_DIGIT_INIT(938), \ + _PyLong_DIGIT_INIT(939), \ + _PyLong_DIGIT_INIT(940), \ + _PyLong_DIGIT_INIT(941), \ + _PyLong_DIGIT_INIT(942), \ + _PyLong_DIGIT_INIT(943), \ + _PyLong_DIGIT_INIT(944), \ + _PyLong_DIGIT_INIT(945), \ + _PyLong_DIGIT_INIT(946), \ + _PyLong_DIGIT_INIT(947), \ + _PyLong_DIGIT_INIT(948), \ + _PyLong_DIGIT_INIT(949), \ + _PyLong_DIGIT_INIT(950), \ + _PyLong_DIGIT_INIT(951), \ + _PyLong_DIGIT_INIT(952), \ + _PyLong_DIGIT_INIT(953), \ + _PyLong_DIGIT_INIT(954), \ + _PyLong_DIGIT_INIT(955), \ + _PyLong_DIGIT_INIT(956), \ + _PyLong_DIGIT_INIT(957), \ + _PyLong_DIGIT_INIT(958), \ + _PyLong_DIGIT_INIT(959), \ + _PyLong_DIGIT_INIT(960), \ + _PyLong_DIGIT_INIT(961), \ + _PyLong_DIGIT_INIT(962), \ + _PyLong_DIGIT_INIT(963), \ + _PyLong_DIGIT_INIT(964), \ + _PyLong_DIGIT_INIT(965), \ + _PyLong_DIGIT_INIT(966), \ + _PyLong_DIGIT_INIT(967), \ + _PyLong_DIGIT_INIT(968), \ + _PyLong_DIGIT_INIT(969), \ + _PyLong_DIGIT_INIT(970), \ + _PyLong_DIGIT_INIT(971), \ + _PyLong_DIGIT_INIT(972), \ + _PyLong_DIGIT_INIT(973), \ + _PyLong_DIGIT_INIT(974), \ + _PyLong_DIGIT_INIT(975), \ + _PyLong_DIGIT_INIT(976), \ + _PyLong_DIGIT_INIT(977), \ + _PyLong_DIGIT_INIT(978), \ + _PyLong_DIGIT_INIT(979), \ + _PyLong_DIGIT_INIT(980), \ + _PyLong_DIGIT_INIT(981), \ + _PyLong_DIGIT_INIT(982), \ + _PyLong_DIGIT_INIT(983), \ + _PyLong_DIGIT_INIT(984), \ + _PyLong_DIGIT_INIT(985), \ + _PyLong_DIGIT_INIT(986), \ + _PyLong_DIGIT_INIT(987), \ + _PyLong_DIGIT_INIT(988), \ + _PyLong_DIGIT_INIT(989), \ + _PyLong_DIGIT_INIT(990), \ + _PyLong_DIGIT_INIT(991), \ + _PyLong_DIGIT_INIT(992), \ + _PyLong_DIGIT_INIT(993), \ + _PyLong_DIGIT_INIT(994), \ + _PyLong_DIGIT_INIT(995), \ + _PyLong_DIGIT_INIT(996), \ + _PyLong_DIGIT_INIT(997), \ + _PyLong_DIGIT_INIT(998), \ + _PyLong_DIGIT_INIT(999), \ + _PyLong_DIGIT_INIT(1000), \ + _PyLong_DIGIT_INIT(1001), \ + _PyLong_DIGIT_INIT(1002), \ + _PyLong_DIGIT_INIT(1003), \ + _PyLong_DIGIT_INIT(1004), \ + _PyLong_DIGIT_INIT(1005), \ + _PyLong_DIGIT_INIT(1006), \ + _PyLong_DIGIT_INIT(1007), \ + _PyLong_DIGIT_INIT(1008), \ + _PyLong_DIGIT_INIT(1009), \ + _PyLong_DIGIT_INIT(1010), \ + _PyLong_DIGIT_INIT(1011), \ + _PyLong_DIGIT_INIT(1012), \ + _PyLong_DIGIT_INIT(1013), \ + _PyLong_DIGIT_INIT(1014), \ + _PyLong_DIGIT_INIT(1015), \ + _PyLong_DIGIT_INIT(1016), \ + _PyLong_DIGIT_INIT(1017), \ + _PyLong_DIGIT_INIT(1018), \ + _PyLong_DIGIT_INIT(1019), \ + _PyLong_DIGIT_INIT(1020), \ + _PyLong_DIGIT_INIT(1021), \ + _PyLong_DIGIT_INIT(1022), \ + _PyLong_DIGIT_INIT(1023), \ + _PyLong_DIGIT_INIT(1024), \ } #define _Py_bytes_characters_INIT { \ @@ -553,21 +1321,35 @@ extern "C" { INIT_STR(dot_locals, "."), \ INIT_STR(empty, ""), \ INIT_STR(format, ".format"), \ + INIT_STR(gc, ""), \ INIT_STR(generic_base, ".generic_base"), \ INIT_STR(json_decoder, "json.decoder"), \ INIT_STR(kwdefaults, ".kwdefaults"), \ INIT_STR(list_err, "list index out of range"), \ + INIT_STR(native, ""), \ INIT_STR(str_replace_inf, "1e309"), \ INIT_STR(type_params, ".type_params"), \ INIT_STR(utf_8, "utf-8"), \ } #define _Py_str_identifiers_INIT { \ + INIT_ID(AGEN_CLOSED), \ + INIT_ID(AGEN_CREATED), \ + INIT_ID(AGEN_RUNNING), \ + INIT_ID(AGEN_SUSPENDED), \ INIT_ID(CANCELLED), \ + INIT_ID(CORO_CLOSED), \ + INIT_ID(CORO_CREATED), \ + INIT_ID(CORO_RUNNING), \ + INIT_ID(CORO_SUSPENDED), \ INIT_ID(Emax), \ INIT_ID(Emin), \ INIT_ID(FINISHED), \ INIT_ID(False), \ + INIT_ID(GEN_CLOSED), \ + INIT_ID(GEN_CREATED), \ + INIT_ID(GEN_RUNNING), \ + INIT_ID(GEN_SUSPENDED), \ INIT_ID(JSONDecodeError), \ INIT_ID(PENDING), \ INIT_ID(Py_Repr), \ @@ -662,6 +1444,8 @@ extern "C" { INIT_ID(__iter__), \ INIT_ID(__itruediv__), \ INIT_ID(__ixor__), \ + INIT_ID(__lazy_import__), \ + INIT_ID(__lazy_modules__), \ INIT_ID(__le__), \ INIT_ID(__len__), \ INIT_ID(__length_hint__), \ @@ -790,13 +1574,16 @@ extern "C" { INIT_ID(aclose), \ INIT_ID(add), \ INIT_ID(add_done_callback), \ + INIT_ID(adobe), \ INIT_ID(after_in_child), \ INIT_ID(after_in_parent), \ INIT_ID(alias), \ INIT_ID(align), \ INIT_ID(all), \ + INIT_ID(all_interpreters), \ INIT_ID(all_threads), \ INIT_ID(allow_code), \ + INIT_ID(alphabet), \ INIT_ID(any), \ INIT_ID(append), \ INIT_ID(arg), \ @@ -837,6 +1624,7 @@ extern "C" { INIT_ID(c_parameter_type), \ INIT_ID(c_return), \ INIT_ID(c_stack), \ + INIT_ID(cache_frames), \ INIT_ID(cached_datetime_module), \ INIT_ID(cached_statements), \ INIT_ID(cadata), \ @@ -847,6 +1635,7 @@ extern "C" { INIT_ID(callable), \ INIT_ID(callback), \ INIT_ID(cancel), \ + INIT_ID(canonical), \ INIT_ID(capath), \ INIT_ID(capitals), \ INIT_ID(category), \ @@ -880,9 +1669,11 @@ extern "C" { INIT_ID(co_varnames), \ INIT_ID(code), \ INIT_ID(col_offset), \ + INIT_ID(collector), \ INIT_ID(command), \ INIT_ID(comment_factory), \ INIT_ID(compile_mode), \ + INIT_ID(compression), \ INIT_ID(config), \ INIT_ID(consts), \ INIT_ID(context), \ @@ -931,6 +1722,7 @@ extern "C" { INIT_ID(eager_start), \ INIT_ID(effective_ids), \ INIT_ID(element_factory), \ + INIT_ID(emptyerror), \ INIT_ID(encode), \ INIT_ID(encoding), \ INIT_ID(end), \ @@ -944,7 +1736,9 @@ extern "C" { INIT_ID(event), \ INIT_ID(eventmask), \ INIT_ID(exc), \ + INIT_ID(exc_tb), \ INIT_ID(exc_type), \ + INIT_ID(exc_val), \ INIT_ID(exc_value), \ INIT_ID(excepthook), \ INIT_ID(exception), \ @@ -956,6 +1750,7 @@ extern "C" { INIT_ID(extra_tokens), \ INIT_ID(facility), \ INIT_ID(factory), \ + INIT_ID(fallback), \ INIT_ID(false), \ INIT_ID(family), \ INIT_ID(fanout), \ @@ -978,6 +1773,7 @@ extern "C" { INIT_ID(flags), \ INIT_ID(flush), \ INIT_ID(fold), \ + INIT_ID(foldspaces), \ INIT_ID(follow_symlinks), \ INIT_ID(format), \ INIT_ID(format_spec), \ @@ -988,8 +1784,10 @@ extern "C" { INIT_ID(fromtimestamp), \ INIT_ID(fromutc), \ INIT_ID(fset), \ + INIT_ID(fullerror), \ INIT_ID(func), \ INIT_ID(future), \ + INIT_ID(gc), \ INIT_ID(generation), \ INIT_ID(get), \ INIT_ID(get_debug), \ @@ -1016,6 +1814,7 @@ extern "C" { INIT_ID(ident), \ INIT_ID(identity_hint), \ INIT_ID(ignore), \ + INIT_ID(ignorechars), \ INIT_ID(imag), \ INIT_ID(implieslink), \ INIT_ID(importlib), \ @@ -1094,12 +1893,15 @@ extern "C" { INIT_ID(loop), \ INIT_ID(manual_reset), \ INIT_ID(mapping), \ + INIT_ID(mask), \ INIT_ID(match), \ INIT_ID(max_length), \ + INIT_ID(max_threads), \ INIT_ID(maxdigits), \ INIT_ID(maxevents), \ INIT_ID(maxlen), \ INIT_ID(maxmem), \ + INIT_ID(maxsize), \ INIT_ID(maxsplit), \ INIT_ID(maxvalue), \ INIT_ID(memLevel), \ @@ -1131,6 +1933,7 @@ extern "C" { INIT_ID(name_from), \ INIT_ID(namespace_separator), \ INIT_ID(namespaces), \ + INIT_ID(native), \ INIT_ID(ndigits), \ INIT_ID(nested), \ INIT_ID(new_file_name), \ @@ -1157,6 +1960,7 @@ extern "C" { INIT_ID(only_keys), \ INIT_ID(oparg), \ INIT_ID(opcode), \ + INIT_ID(opcodes), \ INIT_ID(open), \ INIT_ID(opener), \ INIT_ID(operation), \ @@ -1170,6 +1974,8 @@ extern "C" { INIT_ID(outpath), \ INIT_ID(overlapped), \ INIT_ID(owner), \ + INIT_ID(pad), \ + INIT_ID(padded), \ INIT_ID(pages), \ INIT_ID(parameter), \ INIT_ID(parent), \ @@ -1193,12 +1999,16 @@ extern "C" { INIT_ID(print_file_and_line), \ INIT_ID(priority), \ INIT_ID(progress), \ + INIT_ID(progress_callback), \ INIT_ID(progress_routine), \ INIT_ID(proto), \ INIT_ID(protocol), \ INIT_ID(ps1), \ INIT_ID(ps2), \ + INIT_ID(qid), \ + INIT_ID(qualname), \ INIT_ID(query), \ + INIT_ID(queuetype), \ INIT_ID(quotetabs), \ INIT_ID(raw), \ INIT_ID(read), \ @@ -1210,6 +2020,7 @@ extern "C" { INIT_ID(readline), \ INIT_ID(readonly), \ INIT_ID(real), \ + INIT_ID(recursive), \ INIT_ID(reducer_override), \ INIT_ID(registry), \ INIT_ID(rel_tol), \ @@ -1229,6 +2040,7 @@ extern "C" { INIT_ID(reversed), \ INIT_ID(rounding), \ INIT_ID(salt), \ + INIT_ID(sample_interval_us), \ INIT_ID(sched_priority), \ INIT_ID(scheduler), \ INIT_ID(script), \ @@ -1259,6 +2071,7 @@ extern "C" { INIT_ID(size), \ INIT_ID(sizehint), \ INIT_ID(skip_file_prefixes), \ + INIT_ID(skip_non_matching_threads), \ INIT_ID(sleep), \ INIT_ID(sock), \ INIT_ID(sort), \ @@ -1267,9 +2080,12 @@ extern "C" { INIT_ID(spam), \ INIT_ID(src), \ INIT_ID(src_dir_fd), \ + INIT_ID(stack_frames), \ INIT_ID(stacklevel), \ INIT_ID(start), \ + INIT_ID(start_time_us), \ INIT_ID(statement), \ + INIT_ID(stats), \ INIT_ID(status), \ INIT_ID(stderr), \ INIT_ID(stdin), \ @@ -1287,6 +2103,7 @@ extern "C" { INIT_ID(symmetric_difference_update), \ INIT_ID(tabsize), \ INIT_ID(tag), \ + INIT_ID(take_bytes), \ INIT_ID(target), \ INIT_ID(target_is_directory), \ INIT_ID(task), \ @@ -1307,6 +2124,7 @@ extern "C" { INIT_ID(times), \ INIT_ID(timespec), \ INIT_ID(timestamp), \ + INIT_ID(timestamp_us), \ INIT_ID(timetuple), \ INIT_ID(timeunit), \ INIT_ID(top), \ @@ -1324,6 +2142,7 @@ extern "C" { INIT_ID(tzinfo), \ INIT_ID(tzname), \ INIT_ID(uid), \ + INIT_ID(unboundop), \ INIT_ID(unlink), \ INIT_ID(unraisablehook), \ INIT_ID(updates), \ @@ -1344,6 +2163,7 @@ extern "C" { INIT_ID(which), \ INIT_ID(who), \ INIT_ID(withdata), \ + INIT_ID(wrapcol), \ INIT_ID(writable), \ INIT_ID(write), \ INIT_ID(write_through), \ diff --git a/Include/internal/pycore_runtime_structs.h b/Include/internal/pycore_runtime_structs.h index 12164c7fdd9..145e66de998 100644 --- a/Include/internal/pycore_runtime_structs.h +++ b/Include/internal/pycore_runtime_structs.h @@ -31,20 +31,10 @@ struct _pymem_allocators { debug_alloc_api_t obj; } debug; int is_debug_enabled; + int use_hugepages; PyObjectArenaAllocator obj_arena; }; -enum _py_float_format_type { - _py_float_format_unknown, - _py_float_format_ieee_big_endian, - _py_float_format_ieee_little_endian, -}; - -struct _Py_float_runtime_state { - enum _py_float_format_type float_format; - enum _py_float_format_type double_format; -}; - struct pyhash_runtime_state { struct { #ifndef MS_WINDOWS @@ -77,9 +67,7 @@ struct _fileutils_state { struct _parser_runtime_state { #ifdef Py_DEBUG long memo_statistics[_PYPEGEN_NSTATISTICS]; -#ifdef Py_GIL_DISABLED PyMutex mutex; -#endif #else int _not_used; #endif @@ -106,7 +94,7 @@ struct _Py_cached_objects { }; // These would be in pycore_long.h if it weren't for an include cycle. -#define _PY_NSMALLPOSINTS 257 +#define _PY_NSMALLPOSINTS 1025 #define _PY_NSMALLNEGINTS 5 #include "pycore_global_strings.h" // struct _Py_global_strings @@ -170,10 +158,18 @@ struct pyruntimestate { /* Is Python preinitialized? Set to 1 by Py_PreInitialize() */ int preinitialized; - /* Is Python core initialized? Set to 1 by _Py_InitializeCore() */ + /* Is Python core initialized? Set to 1 by _Py_InitializeCore(). + + Use _PyRuntimeState_GetCoreInitialized() and + _PyRuntimeState_SetCoreInitialized() to access it, + don't access it directly. */ int core_initialized; - /* Is Python fully initialized? Set to 1 by Py_Initialize() */ + /* Is Python fully initialized? Set to 1 by Py_Initialize(). + + Use _PyRuntimeState_GetInitialized() and + _PyRuntimeState_SetInitialized() to access it, + don't access it directly. */ int initialized; /* Set by Py_FinalizeEx(). Only reset to NULL if Py_Initialize() @@ -271,15 +267,18 @@ struct pyruntimestate { } audit_hooks; struct _py_object_runtime_state object_state; - struct _Py_float_runtime_state float_state; struct _Py_unicode_runtime_state unicode_state; struct _types_runtime_state types; struct _Py_time_runtime_state time; #if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) - // Used in "Python/emscripten_trampoline.c" to choose between type - // reflection trampoline and EM_JS trampoline. - int (*emscripten_count_args_function)(PyCFunctionWithKeywords func); + // Used in "Python/emscripten_trampoline.c" to choose between wasm-gc + // trampoline and JavaScript trampoline. + PyObject* (*emscripten_trampoline)(int* success, + PyCFunctionWithKeywords func, + PyObject* self, + PyObject* args, + PyObject* kw); #endif /* All the objects that are shared by the runtime's interpreters. */ diff --git a/Include/internal/pycore_semaphore.h b/Include/internal/pycore_semaphore.h index 26953838460..66b4939dcac 100644 --- a/Include/internal/pycore_semaphore.h +++ b/Include/internal/pycore_semaphore.h @@ -46,10 +46,8 @@ typedef struct _PySemaphore { } _PySemaphore; // Puts the current thread to sleep until _PySemaphore_Wakeup() is called. -// If `detach` is true, then the thread will detach/release the GIL while -// sleeping. PyAPI_FUNC(int) -_PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout_ns, int detach); +_PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout_ns); // Wakes up a single thread waiting on sema. Note that _PySemaphore_Wakeup() // can be called before _PySemaphore_Wait(). diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index c4e8f10fe05..ca4a7c216ed 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -50,27 +50,64 @@ extern "C" { CPython refcounting operations on it! */ +#define Py_INT_TAG 3 +#define Py_TAG_INVALID 2 +#define Py_TAG_REFCNT 1 +#define Py_TAG_BITS 3 + +#define Py_TAGGED_SHIFT 2 #if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) -#define Py_TAG_BITS 0 - PyAPI_FUNC(PyObject *) _Py_stackref_get_object(_PyStackRef ref); PyAPI_FUNC(PyObject *) _Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber); -PyAPI_FUNC(_PyStackRef) _Py_stackref_create(PyObject *obj, const char *filename, int linenumber); +PyAPI_FUNC(_PyStackRef) _Py_stackref_create(PyObject *obj, uint16_t flags, const char *filename, int linenumber); PyAPI_FUNC(void) _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber); +PyAPI_FUNC(_PyStackRef) _Py_stackref_get_borrowed_from(_PyStackRef ref, const char *filename, int linenumber); +PyAPI_FUNC(void) _Py_stackref_set_borrowed_from(_PyStackRef ref, _PyStackRef borrowed_from, const char *filename, int linenumber); extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref); static const _PyStackRef PyStackRef_NULL = { .index = 0 }; -static const _PyStackRef PyStackRef_ERROR = { .index = 2 }; +static const _PyStackRef PyStackRef_ERROR = { .index = (1 << Py_TAGGED_SHIFT) }; -// Use the first 3 even numbers for None, True and False. -// Odd numbers are reserved for (tagged) integers -#define PyStackRef_None ((_PyStackRef){ .index = 4 } ) -#define PyStackRef_False ((_PyStackRef){ .index = 6 }) -#define PyStackRef_True ((_PyStackRef){ .index = 8 }) +#define PyStackRef_None ((_PyStackRef){ .index = (2 << Py_TAGGED_SHIFT) } ) +#define _Py_STACKREF_FALSE_INDEX (3 << Py_TAGGED_SHIFT) +#define _Py_STACKREF_TRUE_INDEX (4 << Py_TAGGED_SHIFT) +#define PyStackRef_False ((_PyStackRef){ .index = _Py_STACKREF_FALSE_INDEX }) +#define PyStackRef_True ((_PyStackRef){ .index = _Py_STACKREF_TRUE_INDEX }) -#define INITIAL_STACKREF_INDEX 10 +#define INITIAL_STACKREF_INDEX (5 << Py_TAGGED_SHIFT) + +#define PyStackRef_ZERO_BITS PyStackRef_NULL + +static inline _PyStackRef +PyStackRef_Wrap(void *ptr) +{ + assert(ptr != NULL); +#ifdef Py_DEBUG + assert(((uint64_t)ptr & Py_TAG_BITS) == 0); + return (_PyStackRef){ .index = ((uint64_t)ptr) | Py_TAG_INVALID }; +#else + return (_PyStackRef){ .index = (uint64_t)ptr }; +#endif +} + +static inline void * +PyStackRef_Unwrap(_PyStackRef ref) +{ +#ifdef Py_DEBUG + assert ((ref.index & Py_TAG_BITS) == Py_TAG_INVALID); + return (void *)(ref.index & ~Py_TAG_BITS); +#else + return (void *)(ref.index); +#endif +} + +static inline int +PyStackRef_RefcountOnObject(_PyStackRef ref) +{ + return (ref.index & Py_TAG_REFCNT) == 0; +} static inline int PyStackRef_IsNull(_PyStackRef ref) @@ -81,7 +118,13 @@ PyStackRef_IsNull(_PyStackRef ref) static inline bool PyStackRef_IsError(_PyStackRef ref) { - return ref.index == 2; + return ref.index == (1 << Py_TAGGED_SHIFT); +} + +static inline bool +PyStackRef_IsMalformed(_PyStackRef ref) +{ + return (ref.index & Py_TAG_BITS) == Py_TAG_INVALID; } static inline bool @@ -112,7 +155,7 @@ PyStackRef_IsNone(_PyStackRef ref) static inline bool PyStackRef_IsTaggedInt(_PyStackRef ref) { - return (ref.index & 1) == 1; + return (ref.index & Py_TAG_BITS) == Py_INT_TAG; } static inline PyObject * @@ -123,50 +166,68 @@ _PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumb _Py_stackref_record_borrow(ref, filename, linenumber); return _Py_stackref_get_object(ref); } - #define PyStackRef_AsPyObjectBorrow(REF) _PyStackRef_AsPyObjectBorrow((REF), __FILE__, __LINE__) static inline PyObject * _PyStackRef_AsPyObjectSteal(_PyStackRef ref, const char *filename, int linenumber) { - return _Py_stackref_close(ref, filename, linenumber); + PyObject *obj = _Py_stackref_close(ref, filename, linenumber); + if (PyStackRef_RefcountOnObject(ref)) { + return obj; + } + return Py_NewRef(obj); } #define PyStackRef_AsPyObjectSteal(REF) _PyStackRef_AsPyObjectSteal((REF), __FILE__, __LINE__) static inline _PyStackRef _PyStackRef_FromPyObjectNew(PyObject *obj, const char *filename, int linenumber) { - Py_INCREF(obj); - return _Py_stackref_create(obj, filename, linenumber); + assert(obj != NULL); + uint16_t flags = 0; + if (!_Py_IsImmortal(obj)) { + _Py_INCREF_MORTAL(obj); + } else { + flags = Py_TAG_REFCNT; + } + return _Py_stackref_create(obj, flags, filename, linenumber); } #define PyStackRef_FromPyObjectNew(obj) _PyStackRef_FromPyObjectNew(_PyObject_CAST(obj), __FILE__, __LINE__) static inline _PyStackRef _PyStackRef_FromPyObjectSteal(PyObject *obj, const char *filename, int linenumber) { - return _Py_stackref_create(obj, filename, linenumber); + assert(obj != NULL); + uint16_t flags = 0; + if (_Py_IsImmortal(obj)) { + flags = Py_TAG_REFCNT; + } + return _Py_stackref_create(obj, flags, filename, linenumber); } #define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj), __FILE__, __LINE__) static inline _PyStackRef _PyStackRef_FromPyObjectBorrow(PyObject *obj, const char *filename, int linenumber) { - return _Py_stackref_create(obj, filename, linenumber); + return _Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber); } #define PyStackRef_FromPyObjectBorrow(obj) _PyStackRef_FromPyObjectBorrow(_PyObject_CAST(obj), __FILE__, __LINE__) static inline void _PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber) { + assert(!PyStackRef_IsError(ref)); + assert(!PyStackRef_IsNull(ref)); if (PyStackRef_IsTaggedInt(ref)) { return; } PyObject *obj = _Py_stackref_close(ref, filename, linenumber); - Py_DECREF(obj); + assert(Py_REFCNT(obj) > 0); + if (PyStackRef_RefcountOnObject(ref)) { + Py_DECREF(obj); + } } #define PyStackRef_CLOSE(REF) _PyStackRef_CLOSE((REF), __FILE__, __LINE__) - static inline void _PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber) { @@ -182,31 +243,56 @@ static inline _PyStackRef _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber) { assert(!PyStackRef_IsError(ref)); + assert(!PyStackRef_IsNull(ref)); if (PyStackRef_IsTaggedInt(ref)) { return ref; } - else { - PyObject *obj = _Py_stackref_get_object(ref); + PyObject *obj = _Py_stackref_get_object(ref); + uint16_t flags = 0; + if (PyStackRef_RefcountOnObject(ref)) { Py_INCREF(obj); - return _Py_stackref_create(obj, filename, linenumber); + } else { + flags = Py_TAG_REFCNT; } + _PyStackRef new_ref = _Py_stackref_create(obj, flags, filename, linenumber); + if (flags == Py_TAG_REFCNT && !_Py_IsImmortal(obj)) { + _PyStackRef borrowed_from = _Py_stackref_get_borrowed_from(ref, filename, linenumber); + _Py_stackref_set_borrowed_from(new_ref, borrowed_from, filename, linenumber); + } + return new_ref; } #define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__) -extern void _PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char *filename, int linenumber); +static inline void +_PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char *filename, int linenumber) +{ + assert(!PyStackRef_IsError(ref)); + assert(!PyStackRef_IsNull(ref)); + assert(!PyStackRef_IsTaggedInt(ref)); + PyObject *obj = _Py_stackref_close(ref, filename, linenumber); + assert(Py_REFCNT(obj) > 0); + if (PyStackRef_RefcountOnObject(ref)) { + _Py_DECREF_SPECIALIZED(obj, destruct); + } +} #define PyStackRef_CLOSE_SPECIALIZED(REF, DESTRUCT) _PyStackRef_CLOSE_SPECIALIZED(REF, DESTRUCT, __FILE__, __LINE__) static inline _PyStackRef -PyStackRef_MakeHeapSafe(_PyStackRef ref) +_PyStackRef_Borrow(_PyStackRef ref, const char *filename, int linenumber) { - return ref; -} - -static inline _PyStackRef -PyStackRef_Borrow(_PyStackRef ref) -{ - return PyStackRef_DUP(ref); + assert(!PyStackRef_IsError(ref)); + assert(!PyStackRef_IsNull(ref)); + if (PyStackRef_IsTaggedInt(ref)) { + return ref; + } + PyObject *obj = _Py_stackref_get_object(ref); + _PyStackRef new_ref = _Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber); + if (!_Py_IsImmortal(obj)) { + _Py_stackref_set_borrowed_from(new_ref, ref, filename, linenumber); + } + return new_ref; } +#define PyStackRef_Borrow(REF) _PyStackRef_Borrow((REF), __FILE__, __LINE__) #define PyStackRef_CLEAR(REF) \ do { \ @@ -219,28 +305,56 @@ PyStackRef_Borrow(_PyStackRef ref) static inline _PyStackRef _PyStackRef_FromPyObjectStealMortal(PyObject *obj, const char *filename, int linenumber) { + assert(obj != NULL); assert(!_Py_IsImmortal(obj)); - return _Py_stackref_create(obj, filename, linenumber); + return _Py_stackref_create(obj, 0, filename, linenumber); } #define PyStackRef_FromPyObjectStealMortal(obj) _PyStackRef_FromPyObjectStealMortal(_PyObject_CAST(obj), __FILE__, __LINE__) static inline bool PyStackRef_IsHeapSafe(_PyStackRef ref) { - return true; + if ((ref.index & Py_TAG_BITS) != Py_TAG_REFCNT || PyStackRef_IsNull(ref)) { + // Tagged ints and ERROR are included. + return true; + } + + PyObject *obj = _Py_stackref_get_object(ref); + return _Py_IsImmortal(obj); } +static inline _PyStackRef +_PyStackRef_MakeHeapSafe(_PyStackRef ref, const char *filename, int linenumber) +{ + // Special references that can't be closed. + if (ref.index < INITIAL_STACKREF_INDEX) { + return ref; + } + + bool heap_safe = PyStackRef_IsHeapSafe(ref); + PyObject *obj = _Py_stackref_close(ref, filename, linenumber); + uint16_t flags = 0; + if (heap_safe) { + // Close old ref and create a new one with the same flags. + // This is necessary for correct borrow checking. + flags = ref.index & Py_TAG_BITS; + } else { + Py_INCREF(obj); + } + return _Py_stackref_create(obj, flags, filename, linenumber); +} +#define PyStackRef_MakeHeapSafe(REF) _PyStackRef_MakeHeapSafe(REF, __FILE__, __LINE__) + static inline _PyStackRef _PyStackRef_FromPyObjectNewMortal(PyObject *obj, const char *filename, int linenumber) { + assert(obj != NULL); assert(!_Py_IsStaticImmortal(obj)); Py_INCREF(obj); - return _Py_stackref_create(obj, filename, linenumber); + return _Py_stackref_create(obj, 0, filename, linenumber); } #define PyStackRef_FromPyObjectNewMortal(obj) _PyStackRef_FromPyObjectNewMortal(_PyObject_CAST(obj), __FILE__, __LINE__) -#define PyStackRef_RefcountOnObject(REF) 1 - extern int PyStackRef_Is(_PyStackRef a, _PyStackRef b); extern bool PyStackRef_IsTaggedInt(_PyStackRef ref); @@ -257,13 +371,12 @@ PyStackRef_IsNullOrInt(_PyStackRef ref); #else -#define Py_INT_TAG 3 -#define Py_TAG_INVALID 2 -#define Py_TAG_REFCNT 1 -#define Py_TAG_BITS 3 - static const _PyStackRef PyStackRef_ERROR = { .bits = Py_TAG_INVALID }; +/* For use in the JIT to clear an unused value. + * PyStackRef_ZERO_BITS has no meaning and should not be used other than by the JIT. */ +static const _PyStackRef PyStackRef_ZERO_BITS = { .bits = 0 }; + /* Wrap a pointer in a stack ref. * The resulting stack reference is not safe and should only be used * in the interpreter to pass values from one uop to another. @@ -273,6 +386,7 @@ PyStackRef_Wrap(void *ptr) { assert(ptr != NULL); #ifdef Py_DEBUG + assert(((uintptr_t)ptr & Py_TAG_BITS) == 0); return (_PyStackRef){ .bits = ((uintptr_t)ptr) | Py_TAG_INVALID }; #else return (_PyStackRef){ .bits = (uintptr_t)ptr }; @@ -296,6 +410,12 @@ PyStackRef_IsError(_PyStackRef ref) return ref.bits == Py_TAG_INVALID; } +static inline bool +PyStackRef_IsMalformed(_PyStackRef ref) +{ + return (ref.bits & Py_TAG_BITS) == Py_TAG_INVALID; +} + static inline bool PyStackRef_IsValid(_PyStackRef ref) { @@ -312,8 +432,9 @@ PyStackRef_IsTaggedInt(_PyStackRef i) static inline _PyStackRef PyStackRef_TagInt(intptr_t i) { - assert(Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (i << 2), 2) == i); - return (_PyStackRef){ .bits = ((((uintptr_t)i) << 2) | Py_INT_TAG) }; + assert(Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (intptr_t)(((uintptr_t)i) << Py_TAGGED_SHIFT), + Py_TAGGED_SHIFT) == i); + return (_PyStackRef){ .bits = ((((uintptr_t)i) << Py_TAGGED_SHIFT) | Py_INT_TAG) }; } static inline intptr_t @@ -321,7 +442,7 @@ PyStackRef_UntagInt(_PyStackRef i) { assert(PyStackRef_IsTaggedInt(i)); intptr_t val = (intptr_t)i.bits; - return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, val, 2); + return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, val, Py_TAGGED_SHIFT); } @@ -329,189 +450,10 @@ static inline _PyStackRef PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref) { assert((ref.bits & Py_TAG_BITS) == Py_INT_TAG); // Is tagged int - assert((ref.bits & (~Py_TAG_BITS)) != (INT_MAX & (~Py_TAG_BITS))); // Isn't about to overflow - return (_PyStackRef){ .bits = ref.bits + 4 }; + assert((ref.bits & (~Py_TAG_BITS)) != (INTPTR_MAX & (~Py_TAG_BITS))); // Isn't about to overflow + return (_PyStackRef){ .bits = ref.bits + (1 << Py_TAGGED_SHIFT) }; } -#define PyStackRef_IsDeferredOrTaggedInt(ref) (((ref).bits & Py_TAG_REFCNT) != 0) - -#ifdef Py_GIL_DISABLED - -#define Py_TAG_DEFERRED Py_TAG_REFCNT - -#define Py_TAG_PTR ((uintptr_t)0) - - -static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED}; - -#define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits) -#define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED }) -#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED }) -#define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_DEFERRED }) - -// Checks that mask out the deferred bit in the free threading build. -#define PyStackRef_IsNone(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_None) -#define PyStackRef_IsTrue(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_True) -#define PyStackRef_IsFalse(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_False) - -#define PyStackRef_IsNullOrInt(stackref) (PyStackRef_IsNull(stackref) || PyStackRef_IsTaggedInt(stackref)) - -static inline PyObject * -PyStackRef_AsPyObjectBorrow(_PyStackRef stackref) -{ - assert(!PyStackRef_IsTaggedInt(stackref)); - PyObject *cleared = ((PyObject *)((stackref).bits & (~Py_TAG_BITS))); - return cleared; -} - -#define PyStackRef_IsDeferred(ref) (((ref).bits & Py_TAG_BITS) == Py_TAG_DEFERRED) - -static inline PyObject * -PyStackRef_NotDeferred_AsPyObject(_PyStackRef stackref) -{ - assert(!PyStackRef_IsDeferred(stackref)); - return (PyObject *)stackref.bits; -} - -static inline PyObject * -PyStackRef_AsPyObjectSteal(_PyStackRef stackref) -{ - assert(!PyStackRef_IsNull(stackref)); - if (PyStackRef_IsDeferred(stackref)) { - return Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)); - } - return PyStackRef_AsPyObjectBorrow(stackref); -} - -static inline _PyStackRef -_PyStackRef_FromPyObjectSteal(PyObject *obj) -{ - assert(obj != NULL); - // Make sure we don't take an already tagged value. - assert(((uintptr_t)obj & Py_TAG_BITS) == 0); - return (_PyStackRef){ .bits = (uintptr_t)obj }; -} -# define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj)) - -static inline bool -PyStackRef_IsHeapSafe(_PyStackRef stackref) -{ - if (PyStackRef_IsDeferred(stackref)) { - PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref); - return obj == NULL || _Py_IsImmortal(obj) || _PyObject_HasDeferredRefcount(obj); - } - return true; -} - -static inline _PyStackRef -PyStackRef_MakeHeapSafe(_PyStackRef stackref) -{ - if (PyStackRef_IsHeapSafe(stackref)) { - return stackref; - } - PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref); - return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR }; -} - -static inline _PyStackRef -PyStackRef_FromPyObjectStealMortal(PyObject *obj) -{ - assert(obj != NULL); - assert(!_Py_IsImmortal(obj)); - // Make sure we don't take an already tagged value. - assert(((uintptr_t)obj & Py_TAG_BITS) == 0); - return (_PyStackRef){ .bits = (uintptr_t)obj }; -} - -static inline _PyStackRef -PyStackRef_FromPyObjectNew(PyObject *obj) -{ - // Make sure we don't take an already tagged value. - assert(((uintptr_t)obj & Py_TAG_BITS) == 0); - assert(obj != NULL); - if (_PyObject_HasDeferredRefcount(obj)) { - return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; - } - else { - return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR }; - } -} -#define PyStackRef_FromPyObjectNew(obj) PyStackRef_FromPyObjectNew(_PyObject_CAST(obj)) - -static inline _PyStackRef -PyStackRef_FromPyObjectBorrow(PyObject *obj) -{ - // Make sure we don't take an already tagged value. - assert(((uintptr_t)obj & Py_TAG_BITS) == 0); - assert(obj != NULL); - return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; -} -#define PyStackRef_FromPyObjectBorrow(obj) PyStackRef_FromPyObjectBorrow(_PyObject_CAST(obj)) - -#define PyStackRef_CLOSE(REF) \ - do { \ - _PyStackRef _close_tmp = (REF); \ - assert(!PyStackRef_IsNull(_close_tmp)); \ - if (!PyStackRef_IsDeferredOrTaggedInt(_close_tmp)) { \ - Py_DECREF(PyStackRef_AsPyObjectBorrow(_close_tmp)); \ - } \ - } while (0) - -static inline void -PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct) -{ - (void)destruct; - PyStackRef_CLOSE(ref); -} - -static inline _PyStackRef -PyStackRef_DUP(_PyStackRef stackref) -{ - assert(!PyStackRef_IsNull(stackref)); - if (PyStackRef_IsDeferredOrTaggedInt(stackref)) { - return stackref; - } - Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref)); - return stackref; -} - -static inline _PyStackRef -PyStackRef_Borrow(_PyStackRef stackref) -{ - return (_PyStackRef){ .bits = stackref.bits | Py_TAG_DEFERRED }; -} - -// Convert a possibly deferred reference to a strong reference. -static inline _PyStackRef -PyStackRef_AsStrongReference(_PyStackRef stackref) -{ - return PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(stackref)); -} - -#define PyStackRef_XCLOSE(stackref) \ - do { \ - _PyStackRef _tmp = (stackref); \ - if (!PyStackRef_IsNull(_tmp)) { \ - PyStackRef_CLOSE(_tmp); \ - } \ - } while (0); - -#define PyStackRef_CLEAR(op) \ - do { \ - _PyStackRef *_tmp_op_ptr = &(op); \ - _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \ - if (!PyStackRef_IsNull(_tmp_old_op)) { \ - *_tmp_op_ptr = PyStackRef_NULL; \ - PyStackRef_CLOSE(_tmp_old_op); \ - } \ - } while (0) - -#define PyStackRef_FromPyObjectNewMortal PyStackRef_FromPyObjectNew - -#else // Py_GIL_DISABLED - -// With GIL - /* References to immortal objects always have their tag bit set to Py_TAG_REFCNT * as they can (must) have their reclamation deferred */ @@ -530,13 +472,24 @@ static const _PyStackRef PyStackRef_NULL = { .bits = PyStackRef_NULL_BITS }; #define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_REFCNT }) #define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_REFCNT }) +#ifdef Py_GIL_DISABLED +// Checks that mask out the deferred bit in the free threading build. +#define PyStackRef_IsNone(REF) (((REF).bits & ~Py_TAG_REFCNT) == (uintptr_t)&_Py_NoneStruct) +#define PyStackRef_IsTrue(REF) (((REF).bits & ~Py_TAG_REFCNT) == (uintptr_t)&_Py_TrueStruct) +#define PyStackRef_IsFalse(REF) (((REF).bits & ~Py_TAG_REFCNT) == (uintptr_t)&_Py_FalseStruct) +#else #define PyStackRef_IsTrue(REF) ((REF).bits == (((uintptr_t)&_Py_TrueStruct) | Py_TAG_REFCNT)) #define PyStackRef_IsFalse(REF) ((REF).bits == (((uintptr_t)&_Py_FalseStruct) | Py_TAG_REFCNT)) #define PyStackRef_IsNone(REF) ((REF).bits == (((uintptr_t)&_Py_NoneStruct) | Py_TAG_REFCNT)) +#endif -#ifdef Py_DEBUG +#define PyStackRef_IsNullOrInt(stackref) (PyStackRef_IsNull(stackref) || PyStackRef_IsTaggedInt(stackref)) -static inline void PyStackRef_CheckValid(_PyStackRef ref) { +#if defined(Py_DEBUG) && !defined(Py_GIL_DISABLED) + +static inline void +PyStackRef_CheckValid(_PyStackRef ref) +{ assert(ref.bits != 0); int tag = ref.bits & Py_TAG_BITS; PyObject *obj = BITS_TO_PTR_MASKED(ref); @@ -587,6 +540,8 @@ PyStackRef_Borrow(_PyStackRef ref) static inline PyObject * PyStackRef_AsPyObjectSteal(_PyStackRef ref) { + assert(!PyStackRef_IsNull(ref)); + assert(!PyStackRef_IsTaggedInt(ref)); if (PyStackRef_RefcountOnObject(ref)) { return BITS_TO_PTR(ref); } @@ -599,14 +554,18 @@ static inline _PyStackRef PyStackRef_FromPyObjectSteal(PyObject *obj) { assert(obj != NULL); -#if SIZEOF_VOID_P > 4 - unsigned int tag = obj->ob_flags & Py_TAG_REFCNT; +#ifdef Py_GIL_DISABLED + return (_PyStackRef){ .bits = (uintptr_t)obj }; #else +# if SIZEOF_VOID_P > 4 + unsigned int tag = obj->ob_flags & Py_TAG_REFCNT; +# else unsigned int tag = _Py_IsImmortal(obj) ? Py_TAG_REFCNT : 0; -#endif +# endif _PyStackRef ref = ((_PyStackRef){.bits = ((uintptr_t)(obj)) | tag}); PyStackRef_CheckValid(ref); return ref; +#endif } static inline _PyStackRef @@ -614,7 +573,7 @@ PyStackRef_FromPyObjectStealMortal(PyObject *obj) { assert(obj != NULL); assert(!_Py_IsImmortal(obj)); - _PyStackRef ref = ((_PyStackRef){.bits = ((uintptr_t)(obj)) }); + _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj }; PyStackRef_CheckValid(ref); return ref; } @@ -623,9 +582,15 @@ static inline _PyStackRef _PyStackRef_FromPyObjectNew(PyObject *obj) { assert(obj != NULL); - if (_Py_IsImmortal(obj)) { - return (_PyStackRef){ .bits = ((uintptr_t)obj) | Py_TAG_REFCNT}; +#ifdef Py_GIL_DISABLED + if (_PyObject_HasDeferredRefcount(obj)) { + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT }; } +#else + if (_Py_IsImmortal(obj)) { + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT }; + } +#endif _Py_INCREF_MORTAL(obj); _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj }; PyStackRef_CheckValid(ref); @@ -648,6 +613,7 @@ _PyStackRef_FromPyObjectNewMortal(PyObject *obj) static inline _PyStackRef PyStackRef_FromPyObjectBorrow(PyObject *obj) { + assert(obj != NULL); return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT}; } @@ -670,7 +636,15 @@ PyStackRef_DUP(_PyStackRef ref) static inline bool PyStackRef_IsHeapSafe(_PyStackRef ref) { - return (ref.bits & Py_TAG_BITS) != Py_TAG_REFCNT || ref.bits == PyStackRef_NULL_BITS || _Py_IsImmortal(BITS_TO_PTR_MASKED(ref)); +#ifdef Py_GIL_DISABLED + if ((ref.bits & Py_TAG_BITS) != Py_TAG_REFCNT) { + return true; + } + PyObject *obj = BITS_TO_PTR_MASKED(ref); + return obj == NULL || _PyObject_HasDeferredRefcount(obj); +#else + return (ref.bits & Py_TAG_BITS) != Py_TAG_REFCNT || ref.bits == PyStackRef_NULL_BITS || _Py_IsImmortal(BITS_TO_PTR_MASKED(ref)); +#endif } static inline _PyStackRef @@ -686,6 +660,13 @@ PyStackRef_MakeHeapSafe(_PyStackRef ref) return ref; } +// Convert a possibly deferred reference to a strong reference. +static inline _PyStackRef +PyStackRef_AsStrongReference(_PyStackRef stackref) +{ + return PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(stackref)); +} + #ifdef _WIN32 #define PyStackRef_CLOSE(REF) \ do { \ @@ -703,12 +684,6 @@ PyStackRef_CLOSE(_PyStackRef ref) } #endif -static inline bool -PyStackRef_IsNullOrInt(_PyStackRef ref) -{ - return PyStackRef_IsNull(ref) || PyStackRef_IsTaggedInt(ref); -} - static inline void PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct) { @@ -741,8 +716,6 @@ PyStackRef_XCLOSE(_PyStackRef ref) } while (0) -#endif // Py_GIL_DISABLED - // Note: this is a macro because MSVC (Windows) has trouble inlining it. #define PyStackRef_Is(a, b) (((a).bits & (~Py_TAG_REFCNT)) == ((b).bits & (~Py_TAG_REFCNT))) @@ -799,6 +772,13 @@ _PyThreadState_PushCStackRef(PyThreadState *tstate, _PyCStackRef *ref) ref->ref = PyStackRef_NULL; } +static inline void +_PyThreadState_PushCStackRefNew(PyThreadState *tstate, _PyCStackRef *ref, PyObject *obj) +{ + _PyThreadState_PushCStackRef(tstate, ref); + ref->ref = PyStackRef_FromPyObjectNew(obj); +} + static inline void _PyThreadState_PopCStackRef(PyThreadState *tstate, _PyCStackRef *ref) { @@ -810,13 +790,24 @@ _PyThreadState_PopCStackRef(PyThreadState *tstate, _PyCStackRef *ref) PyStackRef_XCLOSE(ref->ref); } +static inline _PyStackRef +_PyThreadState_PopCStackRefSteal(PyThreadState *tstate, _PyCStackRef *ref) +{ +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + assert(tstate_impl->c_stack_refs == ref); + tstate_impl->c_stack_refs = ref->next; +#endif + return ref->ref; +} + #ifdef Py_GIL_DISABLED static inline int _Py_TryIncrefCompareStackRef(PyObject **src, PyObject *op, _PyStackRef *out) { if (_PyObject_HasDeferredRefcount(op)) { - *out = (_PyStackRef){ .bits = (uintptr_t)op | Py_TAG_DEFERRED }; + *out = (_PyStackRef){ .bits = (uintptr_t)op | Py_TAG_REFCNT }; return 1; } if (_Py_TryIncrefCompare(src, op)) { @@ -856,6 +847,18 @@ _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out) } \ } while (0) +static inline void +_PyStackRef_CloseStack(_PyStackRef *arguments, int total_args) +{ + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } +} + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_stats.h b/Include/internal/pycore_stats.h index 24f239a2135..850e6ea4552 100644 --- a/Include/internal/pycore_stats.h +++ b/Include/internal/pycore_stats.h @@ -15,39 +15,56 @@ extern "C" { #include "pycore_bitutils.h" // _Py_bit_length -#define STAT_INC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name++; } while (0) -#define STAT_DEC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name--; } while (0) -#define OPCODE_EXE_INC(opname) do { if (_Py_stats) _Py_stats->opcode_stats[opname].execution_count++; } while (0) -#define CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.name++; } while (0) -#define OBJECT_STAT_INC(name) do { if (_Py_stats) _Py_stats->object_stats.name++; } while (0) -#define OBJECT_STAT_INC_COND(name, cond) \ - do { if (_Py_stats && cond) _Py_stats->object_stats.name++; } while (0) -#define EVAL_CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.eval_calls[name]++; } while (0) -#define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \ - do { if (_Py_stats && PyFunction_Check(callable)) _Py_stats->call_stats.eval_calls[name]++; } while (0) -#define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0) -#define OPT_STAT_INC(name) do { if (_Py_stats) _Py_stats->optimization_stats.name++; } while (0) -#define OPT_STAT_ADD(name, n) do { if (_Py_stats) _Py_stats->optimization_stats.name += (n); } while (0) -#define UOP_STAT_INC(opname, name) do { if (_Py_stats) { assert(opname < 512); _Py_stats->optimization_stats.opcode[opname].name++; } } while (0) -#define UOP_PAIR_INC(uopcode, lastuop) \ - do { \ - if (lastuop && _Py_stats) { \ - _Py_stats->optimization_stats.opcode[lastuop].pair_count[uopcode]++; \ - } \ - lastuop = uopcode; \ - } while (0) -#define OPT_UNSUPPORTED_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.unsupported_opcode[opname]++; } while (0) -#define OPT_ERROR_IN_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.error_in_opcode[opname]++; } while (0) -#define OPT_HIST(length, name) \ +#define STAT_INC(opname, name) _Py_STATS_EXPR(opcode_stats[opname].specialization.name++) +#define STAT_DEC(opname, name) _Py_STATS_EXPR(opcode_stats[opname].specialization.name--) +#define OPCODE_EXE_INC(opname) _Py_STATS_EXPR(opcode_stats[opname].execution_count++) +#define CALL_STAT_INC(name) _Py_STATS_EXPR(call_stats.name++) +#define OBJECT_STAT_INC(name) _Py_STATS_EXPR(object_stats.name++) +#define OBJECT_STAT_INC_COND(name, cond) _Py_STATS_COND_EXPR(cond, object_stats.name++) +#define EVAL_CALL_STAT_INC(name) _Py_STATS_EXPR(call_stats.eval_calls[name]++) +#define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) _Py_STATS_COND_EXPR(PyFunction_Check(callable), call_stats.eval_calls[name]++) +#define GC_STAT_ADD(gen, name, n) _Py_STATS_EXPR(gc_stats[(gen)].name += (n)) +#define OPT_STAT_INC(name) _Py_STATS_EXPR(optimization_stats.name++) +#define OPT_STAT_ADD(name, n) _Py_STATS_EXPR(optimization_stats.name += (n)) +#define UOP_STAT_INC(opname, name) \ do { \ - if (_Py_stats) { \ - int bucket = _Py_bit_length(length >= 1 ? length - 1 : 0); \ - bucket = (bucket >= _Py_UOP_HIST_SIZE) ? _Py_UOP_HIST_SIZE - 1 : bucket; \ - _Py_stats->optimization_stats.name[bucket]++; \ + PyStats *s = _PyStats_GET(); \ + if (s) { \ + assert(opname < 512); \ + s->optimization_stats.opcode[opname].name++; \ } \ } while (0) -#define RARE_EVENT_STAT_INC(name) do { if (_Py_stats) _Py_stats->rare_event_stats.name++; } while (0) -#define OPCODE_DEFERRED_INC(opname) do { if (_Py_stats && opcode == opname) _Py_stats->opcode_stats[opname].specialization.deferred++; } while (0) +#define UOP_PAIR_INC(uopcode, lastuop) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (lastuop && s) { \ + s->optimization_stats.opcode[lastuop].pair_count[uopcode]++; \ + } \ + lastuop = uopcode; \ + } while (0) +#define OPT_UNSUPPORTED_OPCODE(opname) _Py_STATS_EXPR(optimization_stats.unsupported_opcode[opname]++) +#define OPT_ERROR_IN_OPCODE(opname) _Py_STATS_EXPR(optimization_stats.error_in_opcode[opname]++) +#define OPT_HIST(length, name) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (s) { \ + int bucket = _Py_bit_length(length >= 1 ? length - 1 : 0); \ + bucket = (bucket >= _Py_UOP_HIST_SIZE) ? _Py_UOP_HIST_SIZE - 1 : bucket; \ + s->optimization_stats.name[bucket]++; \ + } \ + } while (0) +#define RARE_EVENT_STAT_INC(name) _Py_STATS_EXPR(rare_event_stats.name++) +#define OPCODE_DEFERRED_INC(opname) _Py_STATS_COND_EXPR(opcode==opname, opcode_stats[opname].specialization.deferred++) + +#ifdef Py_GIL_DISABLED +#define FT_STAT_MUTEX_SLEEP_INC() _Py_STATS_EXPR(ft_stats.mutex_sleeps++) +#define FT_STAT_QSBR_POLL_INC() _Py_STATS_EXPR(ft_stats.qsbr_polls++) +#define FT_STAT_WORLD_STOP_INC() _Py_STATS_EXPR(ft_stats.world_stops++) +#else +#define FT_STAT_MUTEX_SLEEP_INC() +#define FT_STAT_QSBR_POLL_INC() +#define FT_STAT_WORLD_STOP_INC() +#endif // Export for '_opcode' shared extension PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); @@ -71,6 +88,9 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define OPT_HIST(length, name) ((void)0) #define RARE_EVENT_STAT_INC(name) ((void)0) #define OPCODE_DEFERRED_INC(opname) ((void)0) +#define FT_STAT_MUTEX_SLEEP_INC() +#define FT_STAT_QSBR_POLL_INC() +#define FT_STAT_WORLD_STOP_INC() #endif // !Py_STATS @@ -90,6 +110,11 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); RARE_EVENT_INTERP_INC(interp, name); \ } while (0); \ +PyStatus _PyStats_InterpInit(PyInterpreterState *); +bool _PyStats_ThreadInit(PyInterpreterState *, _PyThreadStateImpl *); +void _PyStats_ThreadFini(_PyThreadStateImpl *); +void _PyStats_Attach(_PyThreadStateImpl *); +void _PyStats_Detach(_PyThreadStateImpl *); #ifdef __cplusplus } diff --git a/Include/internal/pycore_strhex.h b/Include/internal/pycore_strhex.h index 225f423912f..656acae960a 100644 --- a/Include/internal/pycore_strhex.h +++ b/Include/internal/pycore_strhex.h @@ -10,28 +10,24 @@ extern "C" { // Returns a str() containing the hex representation of argbuf. // Export for '_hashlib' shared extension. -PyAPI_FUNC(PyObject*) _Py_strhex(const - char* argbuf, - const Py_ssize_t arglen); +PyAPI_FUNC(PyObject *) _Py_strhex(const char *argbuf, Py_ssize_t arglen); // Returns a bytes() containing the ASCII hex representation of argbuf. -extern PyObject* _Py_strhex_bytes( - const char* argbuf, - const Py_ssize_t arglen); +extern PyObject *_Py_strhex_bytes(const char *argbuf, Py_ssize_t arglen); // These variants include support for a separator between every N bytes: -extern PyObject* _Py_strhex_with_sep( - const char* argbuf, - const Py_ssize_t arglen, - PyObject* sep, - const int bytes_per_group); +extern PyObject *_Py_strhex_with_sep( + const char *argbuf, + Py_ssize_t arglen, + PyObject *sep, + Py_ssize_t bytes_per_group); // Export for 'binascii' shared extension -PyAPI_FUNC(PyObject*) _Py_strhex_bytes_with_sep( - const char* argbuf, - const Py_ssize_t arglen, - PyObject* sep, - const int bytes_per_group); +PyAPI_FUNC(PyObject *) _Py_strhex_bytes_with_sep( + const char *argbuf, + Py_ssize_t arglen, + PyObject *sep, + Py_ssize_t bytes_per_group); #ifdef __cplusplus } diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 98099b4a497..c650a94a1ea 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -90,6 +90,7 @@ typedef struct _symtable_entry { PyObject *ste_id; /* int: key in ste_table->st_blocks */ PyObject *ste_symbols; /* dict: variable names to flags */ PyObject *ste_name; /* string: name of current block */ + PyObject *ste_function_name; /* string or NULL: for annotation blocks: name of the corresponding functions */ PyObject *ste_varnames; /* list of function parameters */ PyObject *ste_children; /* list of child blocks */ PyObject *ste_directives;/* locations of global and nonlocal statements */ @@ -126,6 +127,7 @@ typedef struct _symtable_entry { unsigned ste_method : 1; /* true if block is a function block defined in class scope */ unsigned ste_has_conditional_annotations : 1; /* true if block has conditionally executed annotations */ unsigned ste_in_conditional_block : 1; /* set while we are inside a conditionally executed block */ + unsigned ste_in_try_block : 1; /* set while we are inside a try/except block */ unsigned ste_in_unevaluated_annotation : 1; /* set while we are processing an annotation that will not be evaluated */ int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */ _Py_SourceLocation ste_loc; /* source location of block */ @@ -151,7 +153,12 @@ extern int _PySymtable_LookupOptional(struct symtable *, void *, PySTEntryObject extern void _PySymtable_Free(struct symtable *); extern PyObject *_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name); -extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); + +// Export for '_pickle' shared extension +PyAPI_FUNC(PyObject *) +_Py_Mangle(PyObject *, PyObject *); +PyAPI_FUNC(int) +_Py_IsPrivateName(PyObject *); /* Flags for def-use information */ @@ -188,7 +195,8 @@ extern struct symtable* _Py_SymtableStringObjectFlags( const char *str, PyObject *filename, int start, - PyCompilerFlags *flags); + PyCompilerFlags *flags, + PyObject *module); int _PyFuture_FromAST( struct _mod * mod, diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index 23312471c65..b671225ca6e 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -147,11 +147,6 @@ extern int _PyTime_FromSecondsDouble( // Clamp to [PyTime_MIN; PyTime_MAX] on overflow. extern PyTime_t _PyTime_FromMicrosecondsClamp(PyTime_t us); -// Create a timestamp from a Python int object (number of nanoseconds). -// Export for '_lsprof' shared extension. -PyAPI_FUNC(int) _PyTime_FromLong(PyTime_t *t, - PyObject *obj); - // Convert a number of seconds (Python float or int) to a timestamp. // Raise an exception and return -1 on error, return 0 on success. // Export for '_socket' shared extension. @@ -182,10 +177,6 @@ extern PyTime_t _PyTime_As100Nanoseconds(PyTime_t t, _PyTime_round_t round); #endif -// Convert a timestamp (number of nanoseconds) as a Python int object. -// Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(PyObject*) _PyTime_AsLong(PyTime_t t); - #ifndef MS_WINDOWS // Create a timestamp from a timeval structure. // Raise an exception and return -1 on overflow, return 0 on success. diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index a4f125e073d..fbf6bc2c41f 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -61,7 +61,8 @@ extern void _Py_DumpTraceback( extern const char* _Py_DumpTracebackThreads( int fd, PyInterpreterState *interp, - PyThreadState *current_tstate); + PyThreadState *current_tstate, + Py_ssize_t max_threads); /* Write a Unicode object into the file descriptor fd. Encode the string to ASCII using the backslashreplace error handler. @@ -85,7 +86,8 @@ extern void _Py_DumpHexadecimal( uintptr_t value, Py_ssize_t width); -extern PyObject* _PyTraceBack_FromFrame( +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyTraceBack_FromFrame( PyObject *tb_next, PyFrameObject *frame); @@ -103,6 +105,8 @@ extern int _Py_WriteIndent(int, PyObject *); PyAPI_FUNC(void) _Py_InitDumpStack(void); PyAPI_FUNC(void) _Py_DumpStack(int fd); +extern void _Py_DumpTraceback_Init(void); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_tracemalloc.h b/Include/internal/pycore_tracemalloc.h index 572e8025876..9974ea3c414 100644 --- a/Include/internal/pycore_tracemalloc.h +++ b/Include/internal/pycore_tracemalloc.h @@ -21,7 +21,10 @@ struct _PyTraceMalloc_Config { } initialized; /* Is tracemalloc tracing memory allocations? - Variable protected by the TABLES_LOCK(). */ + Variable protected by the TABLES_LOCK() and stored atomically. + Atomic store is used so that it can read without locking for the + general case of checking if tracemalloc is enabled. + */ int tracing; /* limit of the number of frames in a traceback, 1 by default. @@ -30,8 +33,8 @@ struct _PyTraceMalloc_Config { }; -/* Pack the frame_t structure to reduce the memory footprint on 64-bit - architectures: 12 bytes instead of 16. */ +/* Pack the tracemalloc_frame and tracemalloc_traceback structures to reduce + the memory footprint on 64-bit architectures: 12 bytes instead of 16. */ #if defined(_MSC_VER) #pragma pack(push, 4) #endif @@ -46,18 +49,22 @@ tracemalloc_frame { PyObject *filename; unsigned int lineno; }; -#ifdef _MSC_VER -#pragma pack(pop) -#endif -struct tracemalloc_traceback { +struct +#ifdef __GNUC__ +__attribute__((packed)) +#endif +tracemalloc_traceback { Py_uhash_t hash; /* Number of frames stored */ uint16_t nframe; /* Total number of frames the traceback had */ uint16_t total_nframe; - struct tracemalloc_frame frames[1]; + struct tracemalloc_frame frames[]; }; +#ifdef _MSC_VER +#pragma pack(pop) +#endif struct _tracemalloc_runtime_state { @@ -95,7 +102,7 @@ struct _tracemalloc_runtime_state { Protected by TABLES_LOCK(). */ _Py_hashtable_t *domains; - struct tracemalloc_traceback empty_traceback; + struct tracemalloc_traceback *empty_traceback; Py_tss_t reentrant_key; }; diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index bad968428c7..eb2b0c84acd 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -10,9 +10,11 @@ extern "C" { #include "pycore_brc.h" // struct _brc_thread_state #include "pycore_freelist_state.h" // struct _Py_freelists +#include "pycore_interpframe_structs.h" // _PyInterpreterFrame #include "pycore_mimalloc.h" // struct _mimalloc_thread_state #include "pycore_qsbr.h" // struct qsbr - +#include "pycore_uop.h" // struct _PyUOpInstruction +#include "pycore_structs.h" #ifdef Py_GIL_DISABLED struct _gc_thread_state { @@ -21,6 +23,7 @@ struct _gc_thread_state { }; #endif + // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The // PyThreadState fields are exposed as part of the C API, although most fields // are intended to be private. The _PyThreadStateImpl fields not exposed. @@ -28,6 +31,10 @@ typedef struct _PyThreadStateImpl { // semi-public fields are in PyThreadState. PyThreadState base; + // Embedded base frame - sentinel at the bottom of the frame stack. + // Used by profiling/sampling to detect incomplete stack traces. + _PyInterpreterFrame base_frame; + // The reference count field is used to synchronize deallocation of the // thread state during runtime finalization. Py_ssize_t refcount; @@ -37,9 +44,20 @@ typedef struct _PyThreadStateImpl { uintptr_t c_stack_soft_limit; uintptr_t c_stack_hard_limit; + // PyUnstable_ThreadState_ResetStackProtection() values + uintptr_t c_stack_init_base; + uintptr_t c_stack_init_top; + PyObject *asyncio_running_loop; // Strong reference PyObject *asyncio_running_task; // Strong reference + // Distinguishes between yield and return from PyEval_EvalFrame(). + // See gen_send_ex2() in Objects/genobject.c + enum { + GENERATOR_RETURN = 0, + GENERATOR_YIELD = 1, + } generator_return_kind; + /* Head of circular linked-list of all tasks which are instances of `asyncio.Task` or subclasses of it used in `asyncio.all_tasks`. */ @@ -70,12 +88,26 @@ typedef struct _PyThreadStateImpl { // When >1, code objects do not immortalize their non-string constants. int suppress_co_const_immortalization; + +#ifdef Py_STATS + // per-thread stats, will be merged into interp->pystats_struct + PyStats *pystats_struct; // allocated by _PyStats_ThreadInit() #endif +#endif // Py_GIL_DISABLED + #if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) Py_ssize_t reftotal; // this thread's total refcount operations #endif +#if _Py_TIER2 + struct _PyJitTracerState *jit_tracer_state; +#endif +#ifdef Py_GIL_DISABLED + // gh-144438: Add padding to ensure that the fields above don't share a + // cache line with other allocations. + char __padding[64]; +#endif } _PyThreadStateImpl; #ifdef __cplusplus diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index acf1bec4602..bf80f96396e 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -21,11 +21,17 @@ extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *); /* other API */ +PyAPI_FUNC(void) _PyStolenTuple_Free(PyObject *self); + #define _PyTuple_ITEMS(op) _Py_RVALUE(_PyTuple_CAST(op)->ob_item) -PyAPI_FUNC(PyObject *)_PyTuple_FromArray(PyObject *const *, Py_ssize_t); PyAPI_FUNC(PyObject *)_PyTuple_FromStackRefStealOnSuccess(const union _PyStackRef *, Py_ssize_t); PyAPI_FUNC(PyObject *)_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); +PyAPI_FUNC(PyObject *) _PyTuple_BinarySlice(PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyTuple_Concat(PyObject *, PyObject *); + +PyAPI_FUNC(PyObject *) _PyTuple_FromPair(PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyTuple_FromPairSteal(PyObject *, PyObject *); typedef struct { PyObject_HEAD diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 24df69aa93f..8d48cf6605c 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -10,6 +10,7 @@ extern "C" { #include "pycore_interp_structs.h" // managed_static_type_state #include "pycore_moduleobject.h" // PyModuleObject +#include "pycore_structs.h" // _PyStackRef /* state */ @@ -25,6 +26,7 @@ extern "C" { #define _Py_TYPE_VERSION_BYTEARRAY 9 #define _Py_TYPE_VERSION_BYTES 10 #define _Py_TYPE_VERSION_COMPLEX 11 +#define _Py_TYPE_VERSION_FROZENDICT 12 #define _Py_TYPE_VERSION_NEXT 16 @@ -58,7 +60,8 @@ extern void _PyStaticType_FiniBuiltin( extern void _PyStaticType_ClearWeakRefs( PyInterpreterState *interp, PyTypeObject *type); -extern managed_static_type_state * _PyStaticType_GetState( +// Exported for external JIT support +PyAPI_FUNC(managed_static_type_state *) _PyStaticType_GetState( PyInterpreterState *interp, PyTypeObject *type); @@ -90,8 +93,10 @@ _PyType_GetModuleState(PyTypeObject *type) // function PyAPI_FUNC(PyObject *) _PyType_GetDict(PyTypeObject *); +PyAPI_FUNC(PyObject *) _PyType_LookupSubclasses(PyTypeObject *); +PyAPI_FUNC(PyObject *) _PyType_InitSubclasses(PyTypeObject *); + extern PyObject * _PyType_GetBases(PyTypeObject *type); -extern PyObject * _PyType_GetMRO(PyTypeObject *type); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); extern int _PyType_HasSubclasses(PyTypeObject *); @@ -109,6 +114,8 @@ _PyType_IsReady(PyTypeObject *type) extern PyObject* _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int *suppress_missing_attribute); extern PyObject* _Py_type_getattro(PyObject *type, PyObject *name); +extern _PyStackRef _Py_type_getattro_stackref(PyTypeObject *type, PyObject *name, + int *suppress_missing_attribute); extern PyObject* _Py_BaseObject_RichCompare(PyObject* self, PyObject* other, int op); @@ -120,6 +127,10 @@ extern PyTypeObject _PyBufferWrapper_Type; PyAPI_FUNC(PyObject*) _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, PyObject *name, int *meth_found); +extern PyObject *_PySuper_LookupDescr(PyTypeObject *su_type, + PyTypeObject *su_obj_type, + PyObject *name); + extern PyObject* _PyType_GetFullyQualifiedName(PyTypeObject *type, char sep); // Perform the following operation, in a thread-safe way when required by the @@ -146,8 +157,17 @@ typedef int (*_py_validate_type)(PyTypeObject *); // It will verify the ``ty`` through user-defined validation function ``validate``, // and if the validation is passed, it will set the ``tp_version`` as valid // tp_version_tag from the ``ty``. -extern int _PyType_Validate(PyTypeObject *ty, _py_validate_type validate, unsigned int *tp_version); -extern int _PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version); +// Exported for external JIT support +int _PyType_Validate(PyTypeObject *ty, _py_validate_type validate, unsigned int *tp_version); +int _PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version); + +// Precalculates count of non-unique slots and fills wrapperbase.name_count. +extern int _PyType_InitSlotDefs(PyInterpreterState *interp); + +// Like PyType_GetBaseByToken, but does not modify refcounts. +// Cannot fail; arguments must be valid. +PyAPI_FUNC(int) +_PyType_GetBaseByToken_Borrow(PyTypeObject *type, void *token, PyTypeObject **result); #ifdef __cplusplus } diff --git a/Include/internal/pycore_unicodectype.h b/Include/internal/pycore_unicodectype.h new file mode 100644 index 00000000000..523bdb56b09 --- /dev/null +++ b/Include/internal/pycore_unicodectype.h @@ -0,0 +1,25 @@ +#ifndef Py_INTERNAL_UNICODECTYPE_H +#define Py_INTERNAL_UNICODECTYPE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +extern int _PyUnicode_ToLowerFull(Py_UCS4 ch, Py_UCS4 *res); +extern int _PyUnicode_ToTitleFull(Py_UCS4 ch, Py_UCS4 *res); +extern int _PyUnicode_ToUpperFull(Py_UCS4 ch, Py_UCS4 *res); +extern int _PyUnicode_ToFoldedFull(Py_UCS4 ch, Py_UCS4 *res); +extern int _PyUnicode_IsCaseIgnorable(Py_UCS4 ch); +extern int _PyUnicode_IsCased(Py_UCS4 ch); + +// Export for 'unicodedata' shared extension. +PyAPI_FUNC(int) _PyUnicode_IsXidStart(Py_UCS4 ch); +PyAPI_FUNC(int) _PyUnicode_IsXidContinue(Py_UCS4 ch); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_UNICODECTYPE_H */ diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 8dfcaedd5ef..74d84052a2b 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -11,16 +11,109 @@ extern "C" { #include "pycore_fileutils.h" // _Py_error_handler #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI -/* --- Characters Type APIs ----------------------------------------------- */ -extern int _PyUnicode_IsXidStart(Py_UCS4 ch); -extern int _PyUnicode_IsXidContinue(Py_UCS4 ch); -extern int _PyUnicode_ToLowerFull(Py_UCS4 ch, Py_UCS4 *res); -extern int _PyUnicode_ToTitleFull(Py_UCS4 ch, Py_UCS4 *res); -extern int _PyUnicode_ToUpperFull(Py_UCS4 ch, Py_UCS4 *res); -extern int _PyUnicode_ToFoldedFull(Py_UCS4 ch, Py_UCS4 *res); -extern int _PyUnicode_IsCaseIgnorable(Py_UCS4 ch); -extern int _PyUnicode_IsCased(Py_UCS4 ch); +// Maximum code point of Unicode 6.0: 0x10ffff (1,114,111). +#define _Py_MAX_UNICODE 0x10ffff + + +extern int _PyUnicode_IsModifiable(PyObject *unicode); +extern void _PyUnicodeWriter_InitWithBuffer( + _PyUnicodeWriter *writer, + PyObject *buffer); +extern PyObject* _PyUnicode_Result(PyObject *unicode); +extern int _PyUnicode_DecodeUTF8Writer( + _PyUnicodeWriter *writer, + const char *s, + Py_ssize_t size, + _Py_error_handler error_handler, + const char *errors, + Py_ssize_t *consumed); +extern PyObject* _PyUnicode_ResizeCompact( + PyObject *unicode, + Py_ssize_t length); +extern PyObject* _PyUnicode_GetEmpty(void); +PyAPI_FUNC(PyObject*) _PyUnicode_BinarySlice(PyObject *, PyObject *, PyObject *); + + +/* Generic helper macro to convert characters of different types. + from_type and to_type have to be valid type names, begin and end + are pointers to the source characters which should be of type + "from_type *". to is a pointer of type "to_type *" and points to the + buffer where the result characters are written to. */ +#define _PyUnicode_CONVERT_BYTES(from_type, to_type, begin, end, to) \ + do { \ + to_type *_to = (to_type *)(to); \ + const from_type *_iter = (const from_type *)(begin);\ + const from_type *_end = (const from_type *)(end);\ + Py_ssize_t n = (_end) - (_iter); \ + const from_type *_unrolled_end = \ + _iter + _Py_SIZE_ROUND_DOWN(n, 4); \ + while (_iter < (_unrolled_end)) { \ + _to[0] = (to_type) _iter[0]; \ + _to[1] = (to_type) _iter[1]; \ + _to[2] = (to_type) _iter[2]; \ + _to[3] = (to_type) _iter[3]; \ + _iter += 4; _to += 4; \ + } \ + while (_iter < (_end)) \ + *_to++ = (to_type) *_iter++; \ + } while (0) + + +static inline void +_PyUnicode_Fill(int kind, void *data, Py_UCS4 value, + Py_ssize_t start, Py_ssize_t length) +{ + assert(0 <= start); + switch (kind) { + case PyUnicode_1BYTE_KIND: { + assert(value <= 0xff); + Py_UCS1 ch = (unsigned char)value; + Py_UCS1 *to = (Py_UCS1 *)data + start; + memset(to, ch, length); + break; + } + case PyUnicode_2BYTE_KIND: { + assert(value <= 0xffff); + Py_UCS2 ch = (Py_UCS2)value; + Py_UCS2 *to = (Py_UCS2 *)data + start; + const Py_UCS2 *end = to + length; + for (; to < end; ++to) *to = ch; + break; + } + case PyUnicode_4BYTE_KIND: { + assert(value <= _Py_MAX_UNICODE); + Py_UCS4 ch = value; + Py_UCS4 * to = (Py_UCS4 *)data + start; + const Py_UCS4 *end = to + length; + for (; to < end; ++to) *to = ch; + break; + } + default: Py_UNREACHABLE(); + } +} + +static inline int +_PyUnicode_EnsureUnicode(PyObject *obj) +{ + if (!PyUnicode_Check(obj)) { + PyErr_Format(PyExc_TypeError, + "must be str, not %T", obj); + return -1; + } + return 0; +} + +static inline int +_PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch) +{ + assert(ch <= _Py_MAX_UNICODE); + if (_PyUnicodeWriter_Prepare(writer, 1, ch) < 0) + return -1; + PyUnicode_WRITE(writer->kind, writer->data, writer->pos, ch); + writer->pos++; + return 0; +} /* --- Unicode API -------------------------------------------------------- */ @@ -92,8 +185,6 @@ extern int _PyUnicodeWriter_FormatV( extern PyObject* _PyUnicode_EncodeUTF7( PyObject *unicode, /* Unicode object */ - int base64SetO, /* Encode RFC2152 Set O characters in base64 */ - int base64WhiteSpace, /* Encode whitespace (sp, ht, nl, cr) in base64 */ const char *errors); /* error handling */ /* --- UTF-8 Codecs ------------------------------------------------------- */ @@ -217,14 +308,6 @@ PyAPI_FUNC(PyObject*) _PyUnicode_JoinArray( Py_ssize_t seqlen ); -/* Test whether a unicode is equal to ASCII identifier. Return 1 if true, - 0 otherwise. The right argument must be ASCII identifier. - Any error occurs inside will be cleared before return. */ -extern int _PyUnicode_EqualToASCIIId( - PyObject *left, /* Left string */ - _Py_Identifier *right /* Right identifier */ - ); - // Test whether a unicode is equal to ASCII string. Return 1 if true, // 0 otherwise. The right argument must be ASCII-encoded string. // Any error occurs inside will be cleared before return. @@ -242,23 +325,9 @@ extern PyObject* _PyUnicode_XStrip( ); -/* Using explicit passed-in values, insert the thousands grouping - into the string pointed to by buffer. For the argument descriptions, - see Objects/stringlib/localeutil.h */ -extern Py_ssize_t _PyUnicode_InsertThousandsGrouping( - _PyUnicodeWriter *writer, - Py_ssize_t n_buffer, - PyObject *digits, - Py_ssize_t d_pos, - Py_ssize_t n_digits, - Py_ssize_t min_width, - const char *grouping, - PyObject *thousands_sep, - Py_UCS4 *maxchar, - int forward); - /* Dedent a string. - Behaviour is expected to be an exact match of `textwrap.dedent`. + Intended to dedent Python source. Unlike `textwrap.dedent`, this + only supports spaces and tabs and doesn't normalize empty lines. Return a new reference on success, NULL with exception set on error. */ extern PyObject* _PyUnicode_Dedent(PyObject *unicode); diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index e76e603230a..f0fc3c4f5b0 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -12,10 +12,42 @@ extern "C" { static inline void _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { PyObject *string; + string = &_Py_ID(AGEN_CLOSED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(AGEN_CREATED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(AGEN_RUNNING); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(AGEN_SUSPENDED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(CANCELLED); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(CORO_CLOSED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(CORO_CREATED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(CORO_RUNNING); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(CORO_SUSPENDED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(Emax); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -32,6 +64,22 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(GEN_CLOSED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(GEN_CREATED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(GEN_RUNNING); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(GEN_SUSPENDED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(JSONDecodeError); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -408,6 +456,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(__lazy_import__); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(__lazy_modules__); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__le__); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -920,6 +976,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(adobe); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(after_in_child); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -940,6 +1000,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(all_interpreters); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(all_threads); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -948,6 +1012,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(alphabet); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(any); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1108,6 +1176,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(cache_frames); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cached_datetime_module); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1148,6 +1220,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(canonical); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(capath); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1280,6 +1356,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(collector); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(command); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1292,6 +1372,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(compression); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(config); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1484,6 +1568,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(emptyerror); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(encode); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1536,10 +1624,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(exc_tb); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exc_type); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(exc_val); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exc_value); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1584,6 +1680,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(fallback); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(false); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1672,6 +1772,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(foldspaces); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(follow_symlinks); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1712,6 +1816,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(fullerror); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(func); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1720,6 +1828,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(gc); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(generation); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1824,6 +1936,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(ignorechars); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(imag); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2136,6 +2252,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(mask); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(match); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2144,6 +2264,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(max_threads); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxdigits); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2160,6 +2284,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(maxsize); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxsplit); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2284,6 +2412,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(native); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ndigits); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2388,6 +2520,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(opcodes); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(open); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2440,6 +2576,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(pad); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(padded); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pages); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2532,6 +2676,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(progress_callback); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(progress_routine); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2552,10 +2700,22 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(qid); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(qualname); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(query); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(queuetype); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(quotetabs); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2600,6 +2760,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(recursive); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reducer_override); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2676,6 +2840,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(sample_interval_us); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sched_priority); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2796,6 +2964,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(skip_non_matching_threads); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sleep); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2828,6 +3000,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(stack_frames); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(stacklevel); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2836,10 +3012,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(start_time_us); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(statement); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(stats); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(status); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2908,6 +3092,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(take_bytes); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(target); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2988,6 +3176,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(timestamp_us); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(timetuple); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3056,6 +3248,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(unboundop); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(unlink); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3136,6 +3332,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(wrapcol); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(writable); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3196,6 +3396,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(gc); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_STR(anon_null); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3220,6 +3424,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(native); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_STR(anon_setcomp); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h new file mode 100644 index 00000000000..320508e8b7a --- /dev/null +++ b/Include/internal/pycore_uop.h @@ -0,0 +1,69 @@ +#ifndef Py_CORE_UOP_H +#define Py_CORE_UOP_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include +/* Depending on the format, + * the 32 bits between the oparg and operand are: + * UOP_FORMAT_TARGET: + * uint32_t target; + * UOP_FORMAT_JUMP + * uint16_t jump_target; + * uint16_t error_target; + */ +typedef struct _PyUOpInstruction{ + uint16_t opcode:15; + uint16_t format:1; + uint16_t oparg; + union { + uint32_t target; + struct { + uint16_t jump_target; + uint16_t error_target; + }; + }; + uint64_t operand0; // A cache entry + uint64_t operand1; +#ifdef Py_STATS + int32_t fitness; + uint64_t execution_count; +#endif +} _PyUOpInstruction; + +// This is the length of the trace we translate initially. +#if defined(Py_DEBUG) && defined(_Py_JIT) + // With asserts, the stencils are a lot larger +#define UOP_MAX_TRACE_LENGTH 1000 +#else +#define UOP_MAX_TRACE_LENGTH 2500 +#endif + +/* Bloom filter with m = 256 + * https://en.wikipedia.org/wiki/Bloom_filter */ +#ifdef HAVE_GCC_UINT128_T +#define _Py_BLOOM_FILTER_WORDS 2 +typedef __uint128_t _Py_bloom_filter_word_t; +#else +#define _Py_BLOOM_FILTER_WORDS 4 +typedef uint64_t _Py_bloom_filter_word_t; +#endif + +#define _Py_BLOOM_FILTER_BITS_PER_WORD \ + ((int)(sizeof(_Py_bloom_filter_word_t) * 8)) +#define _Py_BLOOM_FILTER_WORD_SHIFT \ + ((sizeof(_Py_bloom_filter_word_t) == 16) ? 7 : 6) + +typedef struct { + _Py_bloom_filter_word_t bits[_Py_BLOOM_FILTER_WORDS]; +} _PyBloomFilter; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_UOP_H */ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 749369a40ae..bd1440a89bd 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -11,27 +11,42 @@ extern "C" { #define _EXIT_TRACE 300 #define _SET_IP 301 -#define _BINARY_OP 302 -#define _BINARY_OP_ADD_FLOAT 303 -#define _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS 304 -#define _BINARY_OP_ADD_INT 305 -#define _BINARY_OP_ADD_UNICODE 306 -#define _BINARY_OP_EXTEND 307 -#define _BINARY_OP_INPLACE_ADD_UNICODE 308 -#define _BINARY_OP_MULTIPLY_FLOAT 309 -#define _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS 310 -#define _BINARY_OP_MULTIPLY_INT 311 -#define _BINARY_OP_SUBSCR_CHECK_FUNC 312 -#define _BINARY_OP_SUBSCR_DICT 313 -#define _BINARY_OP_SUBSCR_INIT_CALL 314 -#define _BINARY_OP_SUBSCR_LIST_INT 315 -#define _BINARY_OP_SUBSCR_LIST_SLICE 316 -#define _BINARY_OP_SUBSCR_STR_INT 317 -#define _BINARY_OP_SUBSCR_TUPLE_INT 318 -#define _BINARY_OP_SUBTRACT_FLOAT 319 -#define _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS 320 -#define _BINARY_OP_SUBTRACT_INT 321 -#define _BINARY_SLICE 322 +#define _ALLOCATE_OBJECT 302 +#define _BINARY_OP 303 +#define _BINARY_OP_ADD_FLOAT 304 +#define _BINARY_OP_ADD_FLOAT_INPLACE 305 +#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT 306 +#define _BINARY_OP_ADD_INT 307 +#define _BINARY_OP_ADD_INT_INPLACE 308 +#define _BINARY_OP_ADD_INT_INPLACE_RIGHT 309 +#define _BINARY_OP_ADD_UNICODE 310 +#define _BINARY_OP_EXTEND 311 +#define _BINARY_OP_INPLACE_ADD_UNICODE 312 +#define _BINARY_OP_MULTIPLY_FLOAT 313 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE 314 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT 315 +#define _BINARY_OP_MULTIPLY_INT 316 +#define _BINARY_OP_MULTIPLY_INT_INPLACE 317 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT 318 +#define _BINARY_OP_SUBSCR_CHECK_FUNC 319 +#define _BINARY_OP_SUBSCR_DICT 320 +#define _BINARY_OP_SUBSCR_DICT_KNOWN_HASH 321 +#define _BINARY_OP_SUBSCR_INIT_CALL 322 +#define _BINARY_OP_SUBSCR_LIST_INT 323 +#define _BINARY_OP_SUBSCR_LIST_SLICE 324 +#define _BINARY_OP_SUBSCR_STR_INT 325 +#define _BINARY_OP_SUBSCR_TUPLE_INT 326 +#define _BINARY_OP_SUBSCR_USTR_INT 327 +#define _BINARY_OP_SUBTRACT_FLOAT 328 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE 329 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT 330 +#define _BINARY_OP_SUBTRACT_INT 331 +#define _BINARY_OP_SUBTRACT_INT_INPLACE 332 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT 333 +#define _BINARY_OP_TRUEDIV_FLOAT 334 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE 335 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT 336 +#define _BINARY_SLICE 337 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST #define _BUILD_MAP BUILD_MAP @@ -40,143 +55,195 @@ extern "C" { #define _BUILD_STRING BUILD_STRING #define _BUILD_TEMPLATE BUILD_TEMPLATE #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL_BUILTIN_CLASS 323 -#define _CALL_BUILTIN_FAST 324 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 325 -#define _CALL_BUILTIN_O 326 -#define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 -#define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 -#define _CALL_ISINSTANCE 327 -#define _CALL_KW_NON_PY 328 -#define _CALL_LEN 329 -#define _CALL_LIST_APPEND 330 -#define _CALL_METHOD_DESCRIPTOR_FAST 331 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 332 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 333 -#define _CALL_METHOD_DESCRIPTOR_O 334 -#define _CALL_NON_PY_GENERAL 335 -#define _CALL_STR_1 336 -#define _CALL_TUPLE_1 337 -#define _CALL_TYPE_1 338 -#define _CHECK_AND_ALLOCATE_OBJECT 339 -#define _CHECK_ATTR_CLASS 340 -#define _CHECK_ATTR_METHOD_LAZY_DICT 341 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 342 +#define _CALL_BUILTIN_CLASS 338 +#define _CALL_BUILTIN_FAST 339 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 340 +#define _CALL_BUILTIN_O 341 +#define _CALL_FUNCTION_EX_NON_PY_GENERAL 342 +#define _CALL_INTRINSIC_1 343 +#define _CALL_INTRINSIC_2 344 +#define _CALL_ISINSTANCE 345 +#define _CALL_KW_NON_PY 346 +#define _CALL_LEN 347 +#define _CALL_LIST_APPEND 348 +#define _CALL_METHOD_DESCRIPTOR_FAST 349 +#define _CALL_METHOD_DESCRIPTOR_FAST_INLINE 350 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 351 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE 352 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 353 +#define _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE 354 +#define _CALL_METHOD_DESCRIPTOR_O 355 +#define _CALL_METHOD_DESCRIPTOR_O_INLINE 356 +#define _CALL_NON_PY_GENERAL 357 +#define _CALL_STR_1 358 +#define _CALL_TUPLE_1 359 +#define _CALL_TYPE_1 360 +#define _CHECK_ATTR_CLASS 361 +#define _CHECK_ATTR_METHOD_LAZY_DICT 362 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 363 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 343 -#define _CHECK_FUNCTION_EXACT_ARGS 344 -#define _CHECK_FUNCTION_VERSION 345 -#define _CHECK_FUNCTION_VERSION_INLINE 346 -#define _CHECK_FUNCTION_VERSION_KW 347 -#define _CHECK_IS_NOT_PY_CALLABLE 348 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 349 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 350 -#define _CHECK_METHOD_VERSION 351 -#define _CHECK_METHOD_VERSION_KW 352 -#define _CHECK_PEP_523 353 -#define _CHECK_PERIODIC 354 -#define _CHECK_PERIODIC_AT_END 355 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 356 -#define _CHECK_RECURSION_REMAINING 357 -#define _CHECK_STACK_SPACE 358 -#define _CHECK_STACK_SPACE_OPERAND 359 -#define _CHECK_VALIDITY 360 -#define _COLD_EXIT 361 -#define _COMPARE_OP 362 -#define _COMPARE_OP_FLOAT 363 -#define _COMPARE_OP_INT 364 -#define _COMPARE_OP_STR 365 -#define _CONTAINS_OP 366 -#define _CONTAINS_OP_DICT 367 -#define _CONTAINS_OP_SET 368 +#define _CHECK_FUNCTION_EXACT_ARGS 364 +#define _CHECK_FUNCTION_VERSION 365 +#define _CHECK_FUNCTION_VERSION_INLINE 366 +#define _CHECK_FUNCTION_VERSION_KW 367 +#define _CHECK_IS_NOT_PY_CALLABLE 368 +#define _CHECK_IS_NOT_PY_CALLABLE_EX 369 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 370 +#define _CHECK_IS_PY_CALLABLE_EX 371 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 372 +#define _CHECK_METHOD_VERSION 373 +#define _CHECK_METHOD_VERSION_KW 374 +#define _CHECK_OBJECT 375 +#define _CHECK_PEP_523 376 +#define _CHECK_PERIODIC 377 +#define _CHECK_PERIODIC_AT_END 378 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 379 +#define _CHECK_RECURSION_LIMIT 380 +#define _CHECK_RECURSION_REMAINING 381 +#define _CHECK_STACK_SPACE 382 +#define _CHECK_STACK_SPACE_OPERAND 383 +#define _CHECK_VALIDITY 384 +#define _COLD_DYNAMIC_EXIT 385 +#define _COLD_EXIT 386 +#define _COMPARE_OP 387 +#define _COMPARE_OP_FLOAT 388 +#define _COMPARE_OP_INT 389 +#define _COMPARE_OP_STR 390 +#define _CONTAINS_OP 391 +#define _CONTAINS_OP_DICT 392 +#define _CONTAINS_OP_SET 393 #define _CONVERT_VALUE CONVERT_VALUE -#define _COPY 369 -#define _COPY_1 370 -#define _COPY_2 371 -#define _COPY_3 372 +#define _COPY 394 +#define _COPY_1 395 +#define _COPY_2 396 +#define _COPY_3 397 #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 373 +#define _CREATE_INIT_FRAME 398 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 374 -#define _DICT_MERGE DICT_MERGE -#define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 375 -#define _DO_CALL_FUNCTION_EX 376 -#define _DO_CALL_KW 377 +#define _DEOPT 399 +#define _DICT_MERGE 400 +#define _DICT_UPDATE 401 +#define _DO_CALL 402 +#define _DO_CALL_FUNCTION_EX 403 +#define _DO_CALL_KW 404 +#define _DYNAMIC_EXIT 405 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 378 +#define _ERROR_POP_N 406 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 379 -#define _EXPAND_METHOD_KW 380 -#define _FATAL_ERROR 381 +#define _EXPAND_METHOD 407 +#define _EXPAND_METHOD_KW 408 +#define _FATAL_ERROR 409 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 382 -#define _FOR_ITER_GEN_FRAME 383 -#define _FOR_ITER_TIER_TWO 384 +#define _FOR_ITER 410 +#define _FOR_ITER_GEN_FRAME 411 +#define _FOR_ITER_TIER_TWO 412 +#define _FOR_ITER_VIRTUAL 413 +#define _FOR_ITER_VIRTUAL_TIER_TWO 414 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE -#define _GET_ITER GET_ITER +#define _GET_ITER 415 +#define _GET_ITER_TRAD 416 #define _GET_LEN GET_LEN -#define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 385 -#define _GUARD_CALLABLE_ISINSTANCE 386 -#define _GUARD_CALLABLE_LEN 387 -#define _GUARD_CALLABLE_LIST_APPEND 388 -#define _GUARD_CALLABLE_STR_1 389 -#define _GUARD_CALLABLE_TUPLE_1 390 -#define _GUARD_CALLABLE_TYPE_1 391 -#define _GUARD_DORV_NO_DICT 392 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 393 -#define _GUARD_GLOBALS_VERSION 394 -#define _GUARD_IS_FALSE_POP 395 -#define _GUARD_IS_NONE_POP 396 -#define _GUARD_IS_NOT_NONE_POP 397 -#define _GUARD_IS_TRUE_POP 398 -#define _GUARD_KEYS_VERSION 399 -#define _GUARD_NOS_DICT 400 -#define _GUARD_NOS_FLOAT 401 -#define _GUARD_NOS_INT 402 -#define _GUARD_NOS_LIST 403 -#define _GUARD_NOS_NOT_NULL 404 -#define _GUARD_NOS_NULL 405 -#define _GUARD_NOS_OVERFLOWED 406 -#define _GUARD_NOS_TUPLE 407 -#define _GUARD_NOS_UNICODE 408 -#define _GUARD_NOT_EXHAUSTED_LIST 409 -#define _GUARD_NOT_EXHAUSTED_RANGE 410 -#define _GUARD_NOT_EXHAUSTED_TUPLE 411 -#define _GUARD_THIRD_NULL 412 -#define _GUARD_TOS_ANY_SET 413 -#define _GUARD_TOS_DICT 414 -#define _GUARD_TOS_FLOAT 415 -#define _GUARD_TOS_INT 416 -#define _GUARD_TOS_LIST 417 -#define _GUARD_TOS_OVERFLOWED 418 -#define _GUARD_TOS_SLICE 419 -#define _GUARD_TOS_TUPLE 420 -#define _GUARD_TOS_UNICODE 421 -#define _GUARD_TYPE_VERSION 422 -#define _GUARD_TYPE_VERSION_AND_LOCK 423 -#define _HANDLE_PENDING_AND_DEOPT 424 +#define _GUARD_BINARY_OP_EXTEND 417 +#define _GUARD_BINARY_OP_EXTEND_LHS 418 +#define _GUARD_BINARY_OP_EXTEND_RHS 419 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS 420 +#define _GUARD_BIT_IS_SET_POP 421 +#define _GUARD_BIT_IS_SET_POP_4 422 +#define _GUARD_BIT_IS_SET_POP_5 423 +#define _GUARD_BIT_IS_SET_POP_6 424 +#define _GUARD_BIT_IS_SET_POP_7 425 +#define _GUARD_BIT_IS_UNSET_POP 426 +#define _GUARD_BIT_IS_UNSET_POP_4 427 +#define _GUARD_BIT_IS_UNSET_POP_5 428 +#define _GUARD_BIT_IS_UNSET_POP_6 429 +#define _GUARD_BIT_IS_UNSET_POP_7 430 +#define _GUARD_CALLABLE_BUILTIN_CLASS 431 +#define _GUARD_CALLABLE_BUILTIN_FAST 432 +#define _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS 433 +#define _GUARD_CALLABLE_BUILTIN_O 434 +#define _GUARD_CALLABLE_ISINSTANCE 435 +#define _GUARD_CALLABLE_LEN 436 +#define _GUARD_CALLABLE_LIST_APPEND 437 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST 438 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 439 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS 440 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_O 441 +#define _GUARD_CALLABLE_STR_1 442 +#define _GUARD_CALLABLE_TUPLE_1 443 +#define _GUARD_CALLABLE_TYPE_1 444 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR 445 +#define _GUARD_CODE_VERSION_RETURN_VALUE 446 +#define _GUARD_CODE_VERSION_YIELD_VALUE 447 +#define _GUARD_CODE_VERSION__PUSH_FRAME 448 +#define _GUARD_DORV_NO_DICT 449 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 450 +#define _GUARD_GLOBALS_VERSION 451 +#define _GUARD_IP_RETURN_GENERATOR 452 +#define _GUARD_IP_RETURN_VALUE 453 +#define _GUARD_IP_YIELD_VALUE 454 +#define _GUARD_IP__PUSH_FRAME 455 +#define _GUARD_IS_FALSE_POP 456 +#define _GUARD_IS_NONE_POP 457 +#define _GUARD_IS_NOT_NONE_POP 458 +#define _GUARD_IS_TRUE_POP 459 +#define _GUARD_ITERATOR 460 +#define _GUARD_ITER_VIRTUAL 461 +#define _GUARD_KEYS_VERSION 462 +#define _GUARD_LOAD_SUPER_ATTR_METHOD 463 +#define _GUARD_NOS_ANY_DICT 464 +#define _GUARD_NOS_COMPACT_ASCII 465 +#define _GUARD_NOS_DICT 466 +#define _GUARD_NOS_FLOAT 467 +#define _GUARD_NOS_INT 468 +#define _GUARD_NOS_ITER_VIRTUAL 469 +#define _GUARD_NOS_LIST 470 +#define _GUARD_NOS_NOT_NULL 471 +#define _GUARD_NOS_NULL 472 +#define _GUARD_NOS_OVERFLOWED 473 +#define _GUARD_NOS_TUPLE 474 +#define _GUARD_NOS_TYPE_VERSION 475 +#define _GUARD_NOS_UNICODE 476 +#define _GUARD_NOT_EXHAUSTED_LIST 477 +#define _GUARD_NOT_EXHAUSTED_RANGE 478 +#define _GUARD_NOT_EXHAUSTED_TUPLE 479 +#define _GUARD_THIRD_NULL 480 +#define _GUARD_TOS_ANY_DICT 481 +#define _GUARD_TOS_ANY_SET 482 +#define _GUARD_TOS_DICT 483 +#define _GUARD_TOS_FLOAT 484 +#define _GUARD_TOS_FROZENDICT 485 +#define _GUARD_TOS_FROZENSET 486 +#define _GUARD_TOS_INT 487 +#define _GUARD_TOS_LIST 488 +#define _GUARD_TOS_OVERFLOWED 489 +#define _GUARD_TOS_SET 490 +#define _GUARD_TOS_SLICE 491 +#define _GUARD_TOS_TUPLE 492 +#define _GUARD_TOS_UNICODE 493 +#define _GUARD_TYPE 494 +#define _GUARD_TYPE_VERSION 495 +#define _GUARD_TYPE_VERSION_LOCKED 496 +#define _HANDLE_PENDING_AND_DEOPT 497 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 425 -#define _INIT_CALL_PY_EXACT_ARGS 426 -#define _INIT_CALL_PY_EXACT_ARGS_0 427 -#define _INIT_CALL_PY_EXACT_ARGS_1 428 -#define _INIT_CALL_PY_EXACT_ARGS_2 429 -#define _INIT_CALL_PY_EXACT_ARGS_3 430 -#define _INIT_CALL_PY_EXACT_ARGS_4 431 -#define _INSERT_NULL 432 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 498 +#define _INIT_CALL_PY_EXACT_ARGS 499 +#define _INIT_CALL_PY_EXACT_ARGS_0 500 +#define _INIT_CALL_PY_EXACT_ARGS_1 501 +#define _INIT_CALL_PY_EXACT_ARGS_2 502 +#define _INIT_CALL_PY_EXACT_ARGS_3 503 +#define _INIT_CALL_PY_EXACT_ARGS_4 504 +#define _INSERT_NULL 505 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -186,177 +253,1167 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 433 -#define _IS_OP IS_OP -#define _ITER_CHECK_LIST 434 -#define _ITER_CHECK_RANGE 435 -#define _ITER_CHECK_TUPLE 436 -#define _ITER_JUMP_LIST 437 -#define _ITER_JUMP_RANGE 438 -#define _ITER_JUMP_TUPLE 439 -#define _ITER_NEXT_LIST 440 -#define _ITER_NEXT_LIST_TIER_TWO 441 -#define _ITER_NEXT_RANGE 442 -#define _ITER_NEXT_TUPLE 443 -#define _JUMP_TO_TOP 444 +#define _IS_NONE 506 +#define _IS_OP 507 +#define _ITER_CHECK_LIST 508 +#define _ITER_CHECK_RANGE 509 +#define _ITER_CHECK_TUPLE 510 +#define _ITER_JUMP_LIST 511 +#define _ITER_JUMP_RANGE 512 +#define _ITER_JUMP_TUPLE 513 +#define _ITER_NEXT_LIST 514 +#define _ITER_NEXT_LIST_TIER_TWO 515 +#define _ITER_NEXT_RANGE 516 +#define _ITER_NEXT_TUPLE 517 +#define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT +#define _JUMP_TO_TOP 518 #define _LIST_APPEND LIST_APPEND -#define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 445 -#define _LOAD_ATTR_CLASS 446 -#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 447 -#define _LOAD_ATTR_METHOD_LAZY_DICT 448 -#define _LOAD_ATTR_METHOD_NO_DICT 449 -#define _LOAD_ATTR_METHOD_WITH_VALUES 450 -#define _LOAD_ATTR_MODULE 451 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 452 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 453 -#define _LOAD_ATTR_PROPERTY_FRAME 454 -#define _LOAD_ATTR_SLOT 455 -#define _LOAD_ATTR_WITH_HINT 456 +#define _LIST_EXTEND 519 +#define _LOAD_ATTR 520 +#define _LOAD_ATTR_CLASS 521 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME 522 +#define _LOAD_ATTR_INSTANCE_VALUE 523 +#define _LOAD_ATTR_METHOD_LAZY_DICT 524 +#define _LOAD_ATTR_METHOD_NO_DICT 525 +#define _LOAD_ATTR_METHOD_WITH_VALUES 526 +#define _LOAD_ATTR_MODULE 527 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 528 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 529 +#define _LOAD_ATTR_PROPERTY_FRAME 530 +#define _LOAD_ATTR_SLOT 531 +#define _LOAD_ATTR_WITH_HINT 532 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 457 +#define _LOAD_BYTECODE 533 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 458 -#define _LOAD_CONST_INLINE_BORROW 459 -#define _LOAD_CONST_UNDER_INLINE 460 -#define _LOAD_CONST_UNDER_INLINE_BORROW 461 +#define _LOAD_CONST_INLINE 534 +#define _LOAD_CONST_INLINE_BORROW 535 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 462 -#define _LOAD_FAST_0 463 -#define _LOAD_FAST_1 464 -#define _LOAD_FAST_2 465 -#define _LOAD_FAST_3 466 -#define _LOAD_FAST_4 467 -#define _LOAD_FAST_5 468 -#define _LOAD_FAST_6 469 -#define _LOAD_FAST_7 470 +#define _LOAD_FAST 536 +#define _LOAD_FAST_0 537 +#define _LOAD_FAST_1 538 +#define _LOAD_FAST_2 539 +#define _LOAD_FAST_3 540 +#define _LOAD_FAST_4 541 +#define _LOAD_FAST_5 542 +#define _LOAD_FAST_6 543 +#define _LOAD_FAST_7 544 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 471 -#define _LOAD_FAST_BORROW_0 472 -#define _LOAD_FAST_BORROW_1 473 -#define _LOAD_FAST_BORROW_2 474 -#define _LOAD_FAST_BORROW_3 475 -#define _LOAD_FAST_BORROW_4 476 -#define _LOAD_FAST_BORROW_5 477 -#define _LOAD_FAST_BORROW_6 478 -#define _LOAD_FAST_BORROW_7 479 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW +#define _LOAD_FAST_BORROW 545 +#define _LOAD_FAST_BORROW_0 546 +#define _LOAD_FAST_BORROW_1 547 +#define _LOAD_FAST_BORROW_2 548 +#define _LOAD_FAST_BORROW_3 549 +#define _LOAD_FAST_BORROW_4 550 +#define _LOAD_FAST_BORROW_5 551 +#define _LOAD_FAST_BORROW_6 552 +#define _LOAD_FAST_BORROW_7 553 #define _LOAD_FAST_CHECK LOAD_FAST_CHECK -#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 480 -#define _LOAD_GLOBAL_BUILTINS 481 -#define _LOAD_GLOBAL_MODULE 482 +#define _LOAD_GLOBAL 554 +#define _LOAD_GLOBAL_BUILTINS 555 +#define _LOAD_GLOBAL_MODULE 556 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 483 -#define _LOAD_SMALL_INT_0 484 -#define _LOAD_SMALL_INT_1 485 -#define _LOAD_SMALL_INT_2 486 -#define _LOAD_SMALL_INT_3 487 -#define _LOAD_SPECIAL 488 +#define _LOAD_SMALL_INT 557 +#define _LOAD_SMALL_INT_0 558 +#define _LOAD_SMALL_INT_1 559 +#define _LOAD_SMALL_INT_2 560 +#define _LOAD_SMALL_INT_3 561 +#define _LOAD_SPECIAL 562 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR -#define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 489 +#define _LOAD_SUPER_ATTR_METHOD 563 +#define _LOCK_OBJECT 564 +#define _MAKE_CALLARGS_A_TUPLE 565 #define _MAKE_CELL MAKE_CELL -#define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 490 +#define _MAKE_FUNCTION 566 +#define _MAKE_HEAP_SAFE 567 +#define _MAKE_WARM 568 #define _MAP_ADD MAP_ADD -#define _MATCH_CLASS MATCH_CLASS +#define _MATCH_CLASS 569 #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 491 -#define _MAYBE_EXPAND_METHOD_KW 492 -#define _MONITOR_CALL 493 -#define _MONITOR_CALL_KW 494 -#define _MONITOR_JUMP_BACKWARD 495 -#define _MONITOR_RESUME 496 +#define _MAYBE_EXPAND_METHOD 570 +#define _MAYBE_EXPAND_METHOD_KW 571 +#define _MONITOR_CALL 572 +#define _MONITOR_CALL_KW 573 +#define _MONITOR_JUMP_BACKWARD 574 +#define _MONITOR_RESUME 575 #define _NOP NOP -#define _POP_CALL 497 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 498 -#define _POP_CALL_ONE 499 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 500 -#define _POP_CALL_TWO 501 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 502 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 503 -#define _POP_JUMP_IF_TRUE 504 +#define _POP_JUMP_IF_FALSE 576 +#define _POP_JUMP_IF_TRUE 577 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 505 -#define _POP_TOP_INT 506 -#define _POP_TOP_LOAD_CONST_INLINE 507 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 508 -#define _POP_TOP_NOP 509 -#define _POP_TOP_UNICODE 510 -#define _POP_TWO 511 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 512 +#define _POP_TOP_FLOAT 578 +#define _POP_TOP_INT 579 +#define _POP_TOP_NOP 580 +#define _POP_TOP_OPARG 581 +#define _POP_TOP_UNICODE 582 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 513 +#define _PUSH_FRAME 583 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 514 -#define _PY_FRAME_GENERAL 515 -#define _PY_FRAME_KW 516 -#define _QUICKEN_RESUME 517 -#define _REPLACE_WITH_TRUE 518 -#define _RESUME_CHECK RESUME_CHECK +#define _PUSH_NULL_CONDITIONAL 584 +#define _PUSH_TAGGED_ZERO 585 +#define _PY_FRAME_EX 586 +#define _PY_FRAME_GENERAL 587 +#define _PY_FRAME_KW 588 +#define _RECORD_3OS_GEN_FUNC 589 +#define _RECORD_4OS 590 +#define _RECORD_BOUND_METHOD 591 +#define _RECORD_CALLABLE 592 +#define _RECORD_CALLABLE_KW 593 +#define _RECORD_CODE 594 +#define _RECORD_NOS 595 +#define _RECORD_NOS_GEN_FUNC 596 +#define _RECORD_NOS_TYPE 597 +#define _RECORD_TOS 598 +#define _RECORD_TOS_TYPE 599 +#define _REPLACE_WITH_TRUE 600 +#define _RESUME_CHECK 601 #define _RETURN_GENERATOR RETURN_GENERATOR -#define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 519 -#define _SEND 520 -#define _SEND_GEN_FRAME 521 +#define _RETURN_VALUE 602 +#define _RROT_3 603 +#define _SAVE_RETURN_OFFSET 604 +#define _SEND 605 +#define _SEND_GEN_FRAME 606 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE -#define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 522 -#define _STORE_ATTR 523 -#define _STORE_ATTR_INSTANCE_VALUE 524 -#define _STORE_ATTR_SLOT 525 -#define _STORE_ATTR_WITH_HINT 526 +#define _SET_UPDATE 607 +#define _SPILL_OR_RELOAD 608 +#define _START_EXECUTOR 609 +#define _STORE_ATTR 610 +#define _STORE_ATTR_INSTANCE_VALUE 611 +#define _STORE_ATTR_SLOT 612 +#define _STORE_ATTR_WITH_HINT 613 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 527 -#define _STORE_FAST_0 528 -#define _STORE_FAST_1 529 -#define _STORE_FAST_2 530 -#define _STORE_FAST_3 531 -#define _STORE_FAST_4 532 -#define _STORE_FAST_5 533 -#define _STORE_FAST_6 534 -#define _STORE_FAST_7 535 -#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST -#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 536 -#define _STORE_SUBSCR 537 -#define _STORE_SUBSCR_DICT 538 -#define _STORE_SUBSCR_LIST_INT 539 -#define _SWAP 540 -#define _SWAP_2 541 -#define _SWAP_3 542 -#define _TIER2_RESUME_CHECK 543 -#define _TO_BOOL 544 +#define _STORE_SLICE 614 +#define _STORE_SUBSCR 615 +#define _STORE_SUBSCR_DICT 616 +#define _STORE_SUBSCR_DICT_KNOWN_HASH 617 +#define _STORE_SUBSCR_LIST_INT 618 +#define _SWAP 619 +#define _SWAP_2 620 +#define _SWAP_3 621 +#define _SWAP_FAST 622 +#define _SWAP_FAST_0 623 +#define _SWAP_FAST_1 624 +#define _SWAP_FAST_2 625 +#define _SWAP_FAST_3 626 +#define _SWAP_FAST_4 627 +#define _SWAP_FAST_5 628 +#define _SWAP_FAST_6 629 +#define _SWAP_FAST_7 630 +#define _TIER2_RESUME_CHECK 631 +#define _TO_BOOL 632 #define _TO_BOOL_BOOL TO_BOOL_BOOL -#define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 545 +#define _TO_BOOL_INT 633 +#define _TO_BOOL_LIST 634 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 546 -#define _UNARY_INVERT UNARY_INVERT -#define _UNARY_NEGATIVE UNARY_NEGATIVE +#define _TO_BOOL_STR 635 +#define _TRACE_RECORD TRACE_RECORD +#define _UNARY_INVERT 636 +#define _UNARY_NEGATIVE 637 +#define _UNARY_NEGATIVE_FLOAT_INPLACE 638 #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 547 -#define _UNPACK_SEQUENCE_LIST 548 -#define _UNPACK_SEQUENCE_TUPLE 549 -#define _UNPACK_SEQUENCE_TWO_TUPLE 550 +#define _UNPACK_SEQUENCE 639 +#define _UNPACK_SEQUENCE_LIST 640 +#define _UNPACK_SEQUENCE_TUPLE 641 +#define _UNPACK_SEQUENCE_TWO_TUPLE 642 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE 643 +#define _UNPACK_SEQUENCE_UNIQUE_TUPLE 644 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE 645 #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 550 +#define _YIELD_VALUE 646 +#define MAX_UOP_ID 646 +#define _ALLOCATE_OBJECT_r00 647 +#define _BINARY_OP_r23 648 +#define _BINARY_OP_ADD_FLOAT_r03 649 +#define _BINARY_OP_ADD_FLOAT_r13 650 +#define _BINARY_OP_ADD_FLOAT_r23 651 +#define _BINARY_OP_ADD_FLOAT_INPLACE_r03 652 +#define _BINARY_OP_ADD_FLOAT_INPLACE_r13 653 +#define _BINARY_OP_ADD_FLOAT_INPLACE_r23 654 +#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03 655 +#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13 656 +#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23 657 +#define _BINARY_OP_ADD_INT_r03 658 +#define _BINARY_OP_ADD_INT_r13 659 +#define _BINARY_OP_ADD_INT_r23 660 +#define _BINARY_OP_ADD_INT_INPLACE_r03 661 +#define _BINARY_OP_ADD_INT_INPLACE_r13 662 +#define _BINARY_OP_ADD_INT_INPLACE_r23 663 +#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r03 664 +#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r13 665 +#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r23 666 +#define _BINARY_OP_ADD_UNICODE_r03 667 +#define _BINARY_OP_ADD_UNICODE_r13 668 +#define _BINARY_OP_ADD_UNICODE_r23 669 +#define _BINARY_OP_EXTEND_r23 670 +#define _BINARY_OP_INPLACE_ADD_UNICODE_r21 671 +#define _BINARY_OP_MULTIPLY_FLOAT_r03 672 +#define _BINARY_OP_MULTIPLY_FLOAT_r13 673 +#define _BINARY_OP_MULTIPLY_FLOAT_r23 674 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03 675 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13 676 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23 677 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03 678 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13 679 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23 680 +#define _BINARY_OP_MULTIPLY_INT_r03 681 +#define _BINARY_OP_MULTIPLY_INT_r13 682 +#define _BINARY_OP_MULTIPLY_INT_r23 683 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_r03 684 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_r13 685 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_r23 686 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03 687 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13 688 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23 689 +#define _BINARY_OP_SUBSCR_CHECK_FUNC_r23 690 +#define _BINARY_OP_SUBSCR_DICT_r23 691 +#define _BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23 692 +#define _BINARY_OP_SUBSCR_INIT_CALL_r01 693 +#define _BINARY_OP_SUBSCR_INIT_CALL_r11 694 +#define _BINARY_OP_SUBSCR_INIT_CALL_r21 695 +#define _BINARY_OP_SUBSCR_INIT_CALL_r31 696 +#define _BINARY_OP_SUBSCR_LIST_INT_r23 697 +#define _BINARY_OP_SUBSCR_LIST_SLICE_r23 698 +#define _BINARY_OP_SUBSCR_STR_INT_r23 699 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r03 700 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r13 701 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r23 702 +#define _BINARY_OP_SUBSCR_USTR_INT_r23 703 +#define _BINARY_OP_SUBTRACT_FLOAT_r03 704 +#define _BINARY_OP_SUBTRACT_FLOAT_r13 705 +#define _BINARY_OP_SUBTRACT_FLOAT_r23 706 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03 707 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13 708 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23 709 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03 710 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13 711 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23 712 +#define _BINARY_OP_SUBTRACT_INT_r03 713 +#define _BINARY_OP_SUBTRACT_INT_r13 714 +#define _BINARY_OP_SUBTRACT_INT_r23 715 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_r03 716 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_r13 717 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_r23 718 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03 719 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13 720 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23 721 +#define _BINARY_OP_TRUEDIV_FLOAT_r23 722 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03 723 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13 724 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23 725 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03 726 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13 727 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23 728 +#define _BINARY_SLICE_r31 729 +#define _BUILD_INTERPOLATION_r01 730 +#define _BUILD_LIST_r01 731 +#define _BUILD_MAP_r01 732 +#define _BUILD_SET_r01 733 +#define _BUILD_SLICE_r01 734 +#define _BUILD_STRING_r01 735 +#define _BUILD_TEMPLATE_r21 736 +#define _BUILD_TUPLE_r01 737 +#define _CALL_BUILTIN_CLASS_r00 738 +#define _CALL_BUILTIN_FAST_r00 739 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS_r00 740 +#define _CALL_BUILTIN_O_r03 741 +#define _CALL_FUNCTION_EX_NON_PY_GENERAL_r31 742 +#define _CALL_INTRINSIC_1_r12 743 +#define _CALL_INTRINSIC_2_r23 744 +#define _CALL_ISINSTANCE_r31 745 +#define _CALL_KW_NON_PY_r11 746 +#define _CALL_LEN_r33 747 +#define _CALL_LIST_APPEND_r03 748 +#define _CALL_LIST_APPEND_r13 749 +#define _CALL_LIST_APPEND_r23 750 +#define _CALL_LIST_APPEND_r33 751 +#define _CALL_METHOD_DESCRIPTOR_FAST_r00 752 +#define _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00 753 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 754 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00 755 +#define _CALL_METHOD_DESCRIPTOR_NOARGS_r03 756 +#define _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03 757 +#define _CALL_METHOD_DESCRIPTOR_O_r03 758 +#define _CALL_METHOD_DESCRIPTOR_O_INLINE_r03 759 +#define _CALL_NON_PY_GENERAL_r01 760 +#define _CALL_STR_1_r32 761 +#define _CALL_TUPLE_1_r32 762 +#define _CALL_TYPE_1_r02 763 +#define _CALL_TYPE_1_r12 764 +#define _CALL_TYPE_1_r22 765 +#define _CALL_TYPE_1_r32 766 +#define _CHECK_ATTR_CLASS_r01 767 +#define _CHECK_ATTR_CLASS_r11 768 +#define _CHECK_ATTR_CLASS_r22 769 +#define _CHECK_ATTR_CLASS_r33 770 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r01 771 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r11 772 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r22 773 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r33 774 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00 775 +#define _CHECK_EG_MATCH_r22 776 +#define _CHECK_EXC_MATCH_r22 777 +#define _CHECK_FUNCTION_EXACT_ARGS_r00 778 +#define _CHECK_FUNCTION_VERSION_r00 779 +#define _CHECK_FUNCTION_VERSION_INLINE_r00 780 +#define _CHECK_FUNCTION_VERSION_INLINE_r11 781 +#define _CHECK_FUNCTION_VERSION_INLINE_r22 782 +#define _CHECK_FUNCTION_VERSION_INLINE_r33 783 +#define _CHECK_FUNCTION_VERSION_KW_r11 784 +#define _CHECK_IS_NOT_PY_CALLABLE_r00 785 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r03 786 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r13 787 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r23 788 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r33 789 +#define _CHECK_IS_NOT_PY_CALLABLE_KW_r11 790 +#define _CHECK_IS_PY_CALLABLE_EX_r03 791 +#define _CHECK_IS_PY_CALLABLE_EX_r13 792 +#define _CHECK_IS_PY_CALLABLE_EX_r23 793 +#define _CHECK_IS_PY_CALLABLE_EX_r33 794 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r01 795 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r11 796 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r22 797 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r33 798 +#define _CHECK_METHOD_VERSION_r00 799 +#define _CHECK_METHOD_VERSION_KW_r11 800 +#define _CHECK_OBJECT_r00 801 +#define _CHECK_PEP_523_r00 802 +#define _CHECK_PEP_523_r11 803 +#define _CHECK_PEP_523_r22 804 +#define _CHECK_PEP_523_r33 805 +#define _CHECK_PERIODIC_r00 806 +#define _CHECK_PERIODIC_AT_END_r00 807 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00 808 +#define _CHECK_RECURSION_LIMIT_r00 809 +#define _CHECK_RECURSION_LIMIT_r11 810 +#define _CHECK_RECURSION_LIMIT_r22 811 +#define _CHECK_RECURSION_LIMIT_r33 812 +#define _CHECK_RECURSION_REMAINING_r00 813 +#define _CHECK_RECURSION_REMAINING_r11 814 +#define _CHECK_RECURSION_REMAINING_r22 815 +#define _CHECK_RECURSION_REMAINING_r33 816 +#define _CHECK_STACK_SPACE_r00 817 +#define _CHECK_STACK_SPACE_OPERAND_r00 818 +#define _CHECK_STACK_SPACE_OPERAND_r11 819 +#define _CHECK_STACK_SPACE_OPERAND_r22 820 +#define _CHECK_STACK_SPACE_OPERAND_r33 821 +#define _CHECK_VALIDITY_r00 822 +#define _CHECK_VALIDITY_r11 823 +#define _CHECK_VALIDITY_r22 824 +#define _CHECK_VALIDITY_r33 825 +#define _COLD_DYNAMIC_EXIT_r00 826 +#define _COLD_EXIT_r00 827 +#define _COMPARE_OP_r21 828 +#define _COMPARE_OP_FLOAT_r03 829 +#define _COMPARE_OP_FLOAT_r13 830 +#define _COMPARE_OP_FLOAT_r23 831 +#define _COMPARE_OP_INT_r23 832 +#define _COMPARE_OP_STR_r23 833 +#define _CONTAINS_OP_r23 834 +#define _CONTAINS_OP_DICT_r23 835 +#define _CONTAINS_OP_SET_r23 836 +#define _CONVERT_VALUE_r11 837 +#define _COPY_r01 838 +#define _COPY_1_r02 839 +#define _COPY_1_r12 840 +#define _COPY_1_r23 841 +#define _COPY_2_r03 842 +#define _COPY_2_r13 843 +#define _COPY_2_r23 844 +#define _COPY_3_r03 845 +#define _COPY_3_r13 846 +#define _COPY_3_r23 847 +#define _COPY_3_r33 848 +#define _COPY_FREE_VARS_r00 849 +#define _COPY_FREE_VARS_r11 850 +#define _COPY_FREE_VARS_r22 851 +#define _COPY_FREE_VARS_r33 852 +#define _CREATE_INIT_FRAME_r01 853 +#define _DELETE_ATTR_r10 854 +#define _DELETE_DEREF_r00 855 +#define _DELETE_FAST_r00 856 +#define _DELETE_GLOBAL_r00 857 +#define _DELETE_NAME_r00 858 +#define _DELETE_SUBSCR_r20 859 +#define _DEOPT_r00 860 +#define _DEOPT_r10 861 +#define _DEOPT_r20 862 +#define _DEOPT_r30 863 +#define _DICT_MERGE_r11 864 +#define _DICT_UPDATE_r11 865 +#define _DO_CALL_r01 866 +#define _DO_CALL_FUNCTION_EX_r31 867 +#define _DO_CALL_KW_r11 868 +#define _DYNAMIC_EXIT_r00 869 +#define _DYNAMIC_EXIT_r10 870 +#define _DYNAMIC_EXIT_r20 871 +#define _DYNAMIC_EXIT_r30 872 +#define _END_FOR_r10 873 +#define _END_SEND_r31 874 +#define _ERROR_POP_N_r00 875 +#define _EXIT_INIT_CHECK_r10 876 +#define _EXIT_TRACE_r00 877 +#define _EXIT_TRACE_r10 878 +#define _EXIT_TRACE_r20 879 +#define _EXIT_TRACE_r30 880 +#define _EXPAND_METHOD_r00 881 +#define _EXPAND_METHOD_KW_r11 882 +#define _FATAL_ERROR_r00 883 +#define _FATAL_ERROR_r11 884 +#define _FATAL_ERROR_r22 885 +#define _FATAL_ERROR_r33 886 +#define _FORMAT_SIMPLE_r11 887 +#define _FORMAT_WITH_SPEC_r21 888 +#define _FOR_ITER_r23 889 +#define _FOR_ITER_GEN_FRAME_r03 890 +#define _FOR_ITER_GEN_FRAME_r13 891 +#define _FOR_ITER_GEN_FRAME_r23 892 +#define _FOR_ITER_TIER_TWO_r23 893 +#define _FOR_ITER_VIRTUAL_r23 894 +#define _FOR_ITER_VIRTUAL_TIER_TWO_r23 895 +#define _GET_AITER_r11 896 +#define _GET_ANEXT_r12 897 +#define _GET_AWAITABLE_r11 898 +#define _GET_ITER_r12 899 +#define _GET_ITER_TRAD_r12 900 +#define _GET_LEN_r12 901 +#define _GUARD_BINARY_OP_EXTEND_r22 902 +#define _GUARD_BINARY_OP_EXTEND_LHS_r02 903 +#define _GUARD_BINARY_OP_EXTEND_LHS_r12 904 +#define _GUARD_BINARY_OP_EXTEND_LHS_r22 905 +#define _GUARD_BINARY_OP_EXTEND_LHS_r33 906 +#define _GUARD_BINARY_OP_EXTEND_RHS_r02 907 +#define _GUARD_BINARY_OP_EXTEND_RHS_r12 908 +#define _GUARD_BINARY_OP_EXTEND_RHS_r22 909 +#define _GUARD_BINARY_OP_EXTEND_RHS_r33 910 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02 911 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12 912 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22 913 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33 914 +#define _GUARD_BIT_IS_SET_POP_r00 915 +#define _GUARD_BIT_IS_SET_POP_r10 916 +#define _GUARD_BIT_IS_SET_POP_r21 917 +#define _GUARD_BIT_IS_SET_POP_r32 918 +#define _GUARD_BIT_IS_SET_POP_4_r00 919 +#define _GUARD_BIT_IS_SET_POP_4_r10 920 +#define _GUARD_BIT_IS_SET_POP_4_r21 921 +#define _GUARD_BIT_IS_SET_POP_4_r32 922 +#define _GUARD_BIT_IS_SET_POP_5_r00 923 +#define _GUARD_BIT_IS_SET_POP_5_r10 924 +#define _GUARD_BIT_IS_SET_POP_5_r21 925 +#define _GUARD_BIT_IS_SET_POP_5_r32 926 +#define _GUARD_BIT_IS_SET_POP_6_r00 927 +#define _GUARD_BIT_IS_SET_POP_6_r10 928 +#define _GUARD_BIT_IS_SET_POP_6_r21 929 +#define _GUARD_BIT_IS_SET_POP_6_r32 930 +#define _GUARD_BIT_IS_SET_POP_7_r00 931 +#define _GUARD_BIT_IS_SET_POP_7_r10 932 +#define _GUARD_BIT_IS_SET_POP_7_r21 933 +#define _GUARD_BIT_IS_SET_POP_7_r32 934 +#define _GUARD_BIT_IS_UNSET_POP_r00 935 +#define _GUARD_BIT_IS_UNSET_POP_r10 936 +#define _GUARD_BIT_IS_UNSET_POP_r21 937 +#define _GUARD_BIT_IS_UNSET_POP_r32 938 +#define _GUARD_BIT_IS_UNSET_POP_4_r00 939 +#define _GUARD_BIT_IS_UNSET_POP_4_r10 940 +#define _GUARD_BIT_IS_UNSET_POP_4_r21 941 +#define _GUARD_BIT_IS_UNSET_POP_4_r32 942 +#define _GUARD_BIT_IS_UNSET_POP_5_r00 943 +#define _GUARD_BIT_IS_UNSET_POP_5_r10 944 +#define _GUARD_BIT_IS_UNSET_POP_5_r21 945 +#define _GUARD_BIT_IS_UNSET_POP_5_r32 946 +#define _GUARD_BIT_IS_UNSET_POP_6_r00 947 +#define _GUARD_BIT_IS_UNSET_POP_6_r10 948 +#define _GUARD_BIT_IS_UNSET_POP_6_r21 949 +#define _GUARD_BIT_IS_UNSET_POP_6_r32 950 +#define _GUARD_BIT_IS_UNSET_POP_7_r00 951 +#define _GUARD_BIT_IS_UNSET_POP_7_r10 952 +#define _GUARD_BIT_IS_UNSET_POP_7_r21 953 +#define _GUARD_BIT_IS_UNSET_POP_7_r32 954 +#define _GUARD_CALLABLE_BUILTIN_CLASS_r00 955 +#define _GUARD_CALLABLE_BUILTIN_FAST_r00 956 +#define _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00 957 +#define _GUARD_CALLABLE_BUILTIN_O_r00 958 +#define _GUARD_CALLABLE_ISINSTANCE_r03 959 +#define _GUARD_CALLABLE_ISINSTANCE_r13 960 +#define _GUARD_CALLABLE_ISINSTANCE_r23 961 +#define _GUARD_CALLABLE_ISINSTANCE_r33 962 +#define _GUARD_CALLABLE_LEN_r03 963 +#define _GUARD_CALLABLE_LEN_r13 964 +#define _GUARD_CALLABLE_LEN_r23 965 +#define _GUARD_CALLABLE_LEN_r33 966 +#define _GUARD_CALLABLE_LIST_APPEND_r03 967 +#define _GUARD_CALLABLE_LIST_APPEND_r13 968 +#define _GUARD_CALLABLE_LIST_APPEND_r23 969 +#define _GUARD_CALLABLE_LIST_APPEND_r33 970 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00 971 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 972 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00 973 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00 974 +#define _GUARD_CALLABLE_STR_1_r03 975 +#define _GUARD_CALLABLE_STR_1_r13 976 +#define _GUARD_CALLABLE_STR_1_r23 977 +#define _GUARD_CALLABLE_STR_1_r33 978 +#define _GUARD_CALLABLE_TUPLE_1_r03 979 +#define _GUARD_CALLABLE_TUPLE_1_r13 980 +#define _GUARD_CALLABLE_TUPLE_1_r23 981 +#define _GUARD_CALLABLE_TUPLE_1_r33 982 +#define _GUARD_CALLABLE_TYPE_1_r03 983 +#define _GUARD_CALLABLE_TYPE_1_r13 984 +#define _GUARD_CALLABLE_TYPE_1_r23 985 +#define _GUARD_CALLABLE_TYPE_1_r33 986 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r00 987 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r11 988 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r22 989 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r33 990 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r00 991 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r11 992 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r22 993 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r33 994 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r00 995 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r11 996 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r22 997 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r33 998 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r00 999 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r11 1000 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r22 1001 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r33 1002 +#define _GUARD_DORV_NO_DICT_r01 1003 +#define _GUARD_DORV_NO_DICT_r11 1004 +#define _GUARD_DORV_NO_DICT_r22 1005 +#define _GUARD_DORV_NO_DICT_r33 1006 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01 1007 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 1008 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 1009 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 1010 +#define _GUARD_GLOBALS_VERSION_r00 1011 +#define _GUARD_GLOBALS_VERSION_r11 1012 +#define _GUARD_GLOBALS_VERSION_r22 1013 +#define _GUARD_GLOBALS_VERSION_r33 1014 +#define _GUARD_IP_RETURN_GENERATOR_r00 1015 +#define _GUARD_IP_RETURN_GENERATOR_r11 1016 +#define _GUARD_IP_RETURN_GENERATOR_r22 1017 +#define _GUARD_IP_RETURN_GENERATOR_r33 1018 +#define _GUARD_IP_RETURN_VALUE_r00 1019 +#define _GUARD_IP_RETURN_VALUE_r11 1020 +#define _GUARD_IP_RETURN_VALUE_r22 1021 +#define _GUARD_IP_RETURN_VALUE_r33 1022 +#define _GUARD_IP_YIELD_VALUE_r00 1023 +#define _GUARD_IP_YIELD_VALUE_r11 1024 +#define _GUARD_IP_YIELD_VALUE_r22 1025 +#define _GUARD_IP_YIELD_VALUE_r33 1026 +#define _GUARD_IP__PUSH_FRAME_r00 1027 +#define _GUARD_IP__PUSH_FRAME_r11 1028 +#define _GUARD_IP__PUSH_FRAME_r22 1029 +#define _GUARD_IP__PUSH_FRAME_r33 1030 +#define _GUARD_IS_FALSE_POP_r00 1031 +#define _GUARD_IS_FALSE_POP_r10 1032 +#define _GUARD_IS_FALSE_POP_r21 1033 +#define _GUARD_IS_FALSE_POP_r32 1034 +#define _GUARD_IS_NONE_POP_r00 1035 +#define _GUARD_IS_NONE_POP_r10 1036 +#define _GUARD_IS_NONE_POP_r21 1037 +#define _GUARD_IS_NONE_POP_r32 1038 +#define _GUARD_IS_NOT_NONE_POP_r10 1039 +#define _GUARD_IS_TRUE_POP_r00 1040 +#define _GUARD_IS_TRUE_POP_r10 1041 +#define _GUARD_IS_TRUE_POP_r21 1042 +#define _GUARD_IS_TRUE_POP_r32 1043 +#define _GUARD_ITERATOR_r01 1044 +#define _GUARD_ITERATOR_r11 1045 +#define _GUARD_ITERATOR_r22 1046 +#define _GUARD_ITERATOR_r33 1047 +#define _GUARD_ITER_VIRTUAL_r01 1048 +#define _GUARD_ITER_VIRTUAL_r11 1049 +#define _GUARD_ITER_VIRTUAL_r22 1050 +#define _GUARD_ITER_VIRTUAL_r33 1051 +#define _GUARD_KEYS_VERSION_r01 1052 +#define _GUARD_KEYS_VERSION_r11 1053 +#define _GUARD_KEYS_VERSION_r22 1054 +#define _GUARD_KEYS_VERSION_r33 1055 +#define _GUARD_LOAD_SUPER_ATTR_METHOD_r03 1056 +#define _GUARD_LOAD_SUPER_ATTR_METHOD_r13 1057 +#define _GUARD_LOAD_SUPER_ATTR_METHOD_r23 1058 +#define _GUARD_LOAD_SUPER_ATTR_METHOD_r33 1059 +#define _GUARD_NOS_ANY_DICT_r02 1060 +#define _GUARD_NOS_ANY_DICT_r12 1061 +#define _GUARD_NOS_ANY_DICT_r22 1062 +#define _GUARD_NOS_ANY_DICT_r33 1063 +#define _GUARD_NOS_COMPACT_ASCII_r02 1064 +#define _GUARD_NOS_COMPACT_ASCII_r12 1065 +#define _GUARD_NOS_COMPACT_ASCII_r22 1066 +#define _GUARD_NOS_COMPACT_ASCII_r33 1067 +#define _GUARD_NOS_DICT_r02 1068 +#define _GUARD_NOS_DICT_r12 1069 +#define _GUARD_NOS_DICT_r22 1070 +#define _GUARD_NOS_DICT_r33 1071 +#define _GUARD_NOS_FLOAT_r02 1072 +#define _GUARD_NOS_FLOAT_r12 1073 +#define _GUARD_NOS_FLOAT_r22 1074 +#define _GUARD_NOS_FLOAT_r33 1075 +#define _GUARD_NOS_INT_r02 1076 +#define _GUARD_NOS_INT_r12 1077 +#define _GUARD_NOS_INT_r22 1078 +#define _GUARD_NOS_INT_r33 1079 +#define _GUARD_NOS_ITER_VIRTUAL_r02 1080 +#define _GUARD_NOS_ITER_VIRTUAL_r12 1081 +#define _GUARD_NOS_ITER_VIRTUAL_r22 1082 +#define _GUARD_NOS_ITER_VIRTUAL_r33 1083 +#define _GUARD_NOS_LIST_r02 1084 +#define _GUARD_NOS_LIST_r12 1085 +#define _GUARD_NOS_LIST_r22 1086 +#define _GUARD_NOS_LIST_r33 1087 +#define _GUARD_NOS_NOT_NULL_r02 1088 +#define _GUARD_NOS_NOT_NULL_r12 1089 +#define _GUARD_NOS_NOT_NULL_r22 1090 +#define _GUARD_NOS_NOT_NULL_r33 1091 +#define _GUARD_NOS_NULL_r02 1092 +#define _GUARD_NOS_NULL_r12 1093 +#define _GUARD_NOS_NULL_r22 1094 +#define _GUARD_NOS_NULL_r33 1095 +#define _GUARD_NOS_OVERFLOWED_r02 1096 +#define _GUARD_NOS_OVERFLOWED_r12 1097 +#define _GUARD_NOS_OVERFLOWED_r22 1098 +#define _GUARD_NOS_OVERFLOWED_r33 1099 +#define _GUARD_NOS_TUPLE_r02 1100 +#define _GUARD_NOS_TUPLE_r12 1101 +#define _GUARD_NOS_TUPLE_r22 1102 +#define _GUARD_NOS_TUPLE_r33 1103 +#define _GUARD_NOS_TYPE_VERSION_r02 1104 +#define _GUARD_NOS_TYPE_VERSION_r12 1105 +#define _GUARD_NOS_TYPE_VERSION_r22 1106 +#define _GUARD_NOS_TYPE_VERSION_r33 1107 +#define _GUARD_NOS_UNICODE_r02 1108 +#define _GUARD_NOS_UNICODE_r12 1109 +#define _GUARD_NOS_UNICODE_r22 1110 +#define _GUARD_NOS_UNICODE_r33 1111 +#define _GUARD_NOT_EXHAUSTED_LIST_r02 1112 +#define _GUARD_NOT_EXHAUSTED_LIST_r12 1113 +#define _GUARD_NOT_EXHAUSTED_LIST_r22 1114 +#define _GUARD_NOT_EXHAUSTED_LIST_r33 1115 +#define _GUARD_NOT_EXHAUSTED_RANGE_r02 1116 +#define _GUARD_NOT_EXHAUSTED_RANGE_r12 1117 +#define _GUARD_NOT_EXHAUSTED_RANGE_r22 1118 +#define _GUARD_NOT_EXHAUSTED_RANGE_r33 1119 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r02 1120 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r12 1121 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 1122 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 1123 +#define _GUARD_THIRD_NULL_r03 1124 +#define _GUARD_THIRD_NULL_r13 1125 +#define _GUARD_THIRD_NULL_r23 1126 +#define _GUARD_THIRD_NULL_r33 1127 +#define _GUARD_TOS_ANY_DICT_r01 1128 +#define _GUARD_TOS_ANY_DICT_r11 1129 +#define _GUARD_TOS_ANY_DICT_r22 1130 +#define _GUARD_TOS_ANY_DICT_r33 1131 +#define _GUARD_TOS_ANY_SET_r01 1132 +#define _GUARD_TOS_ANY_SET_r11 1133 +#define _GUARD_TOS_ANY_SET_r22 1134 +#define _GUARD_TOS_ANY_SET_r33 1135 +#define _GUARD_TOS_DICT_r01 1136 +#define _GUARD_TOS_DICT_r11 1137 +#define _GUARD_TOS_DICT_r22 1138 +#define _GUARD_TOS_DICT_r33 1139 +#define _GUARD_TOS_FLOAT_r01 1140 +#define _GUARD_TOS_FLOAT_r11 1141 +#define _GUARD_TOS_FLOAT_r22 1142 +#define _GUARD_TOS_FLOAT_r33 1143 +#define _GUARD_TOS_FROZENDICT_r01 1144 +#define _GUARD_TOS_FROZENDICT_r11 1145 +#define _GUARD_TOS_FROZENDICT_r22 1146 +#define _GUARD_TOS_FROZENDICT_r33 1147 +#define _GUARD_TOS_FROZENSET_r01 1148 +#define _GUARD_TOS_FROZENSET_r11 1149 +#define _GUARD_TOS_FROZENSET_r22 1150 +#define _GUARD_TOS_FROZENSET_r33 1151 +#define _GUARD_TOS_INT_r01 1152 +#define _GUARD_TOS_INT_r11 1153 +#define _GUARD_TOS_INT_r22 1154 +#define _GUARD_TOS_INT_r33 1155 +#define _GUARD_TOS_LIST_r01 1156 +#define _GUARD_TOS_LIST_r11 1157 +#define _GUARD_TOS_LIST_r22 1158 +#define _GUARD_TOS_LIST_r33 1159 +#define _GUARD_TOS_OVERFLOWED_r01 1160 +#define _GUARD_TOS_OVERFLOWED_r11 1161 +#define _GUARD_TOS_OVERFLOWED_r22 1162 +#define _GUARD_TOS_OVERFLOWED_r33 1163 +#define _GUARD_TOS_SET_r01 1164 +#define _GUARD_TOS_SET_r11 1165 +#define _GUARD_TOS_SET_r22 1166 +#define _GUARD_TOS_SET_r33 1167 +#define _GUARD_TOS_SLICE_r01 1168 +#define _GUARD_TOS_SLICE_r11 1169 +#define _GUARD_TOS_SLICE_r22 1170 +#define _GUARD_TOS_SLICE_r33 1171 +#define _GUARD_TOS_TUPLE_r01 1172 +#define _GUARD_TOS_TUPLE_r11 1173 +#define _GUARD_TOS_TUPLE_r22 1174 +#define _GUARD_TOS_TUPLE_r33 1175 +#define _GUARD_TOS_UNICODE_r01 1176 +#define _GUARD_TOS_UNICODE_r11 1177 +#define _GUARD_TOS_UNICODE_r22 1178 +#define _GUARD_TOS_UNICODE_r33 1179 +#define _GUARD_TYPE_r01 1180 +#define _GUARD_TYPE_r11 1181 +#define _GUARD_TYPE_r22 1182 +#define _GUARD_TYPE_r33 1183 +#define _GUARD_TYPE_VERSION_r01 1184 +#define _GUARD_TYPE_VERSION_r11 1185 +#define _GUARD_TYPE_VERSION_r22 1186 +#define _GUARD_TYPE_VERSION_r33 1187 +#define _GUARD_TYPE_VERSION_LOCKED_r01 1188 +#define _GUARD_TYPE_VERSION_LOCKED_r11 1189 +#define _GUARD_TYPE_VERSION_LOCKED_r22 1190 +#define _GUARD_TYPE_VERSION_LOCKED_r33 1191 +#define _HANDLE_PENDING_AND_DEOPT_r00 1192 +#define _HANDLE_PENDING_AND_DEOPT_r10 1193 +#define _HANDLE_PENDING_AND_DEOPT_r20 1194 +#define _HANDLE_PENDING_AND_DEOPT_r30 1195 +#define _IMPORT_FROM_r12 1196 +#define _IMPORT_NAME_r21 1197 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 1198 +#define _INIT_CALL_PY_EXACT_ARGS_r01 1199 +#define _INIT_CALL_PY_EXACT_ARGS_0_r01 1200 +#define _INIT_CALL_PY_EXACT_ARGS_1_r01 1201 +#define _INIT_CALL_PY_EXACT_ARGS_2_r01 1202 +#define _INIT_CALL_PY_EXACT_ARGS_3_r01 1203 +#define _INIT_CALL_PY_EXACT_ARGS_4_r01 1204 +#define _INSERT_NULL_r10 1205 +#define _INSTRUMENTED_FOR_ITER_r23 1206 +#define _INSTRUMENTED_INSTRUCTION_r00 1207 +#define _INSTRUMENTED_JUMP_FORWARD_r00 1208 +#define _INSTRUMENTED_JUMP_FORWARD_r11 1209 +#define _INSTRUMENTED_JUMP_FORWARD_r22 1210 +#define _INSTRUMENTED_JUMP_FORWARD_r33 1211 +#define _INSTRUMENTED_LINE_r00 1212 +#define _INSTRUMENTED_NOT_TAKEN_r00 1213 +#define _INSTRUMENTED_NOT_TAKEN_r11 1214 +#define _INSTRUMENTED_NOT_TAKEN_r22 1215 +#define _INSTRUMENTED_NOT_TAKEN_r33 1216 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 1217 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 1218 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 1219 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 1220 +#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 1221 +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 1222 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 1223 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 1224 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 1225 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 1226 +#define _IS_NONE_r11 1227 +#define _IS_OP_r03 1228 +#define _IS_OP_r13 1229 +#define _IS_OP_r23 1230 +#define _ITER_CHECK_LIST_r02 1231 +#define _ITER_CHECK_LIST_r12 1232 +#define _ITER_CHECK_LIST_r22 1233 +#define _ITER_CHECK_LIST_r33 1234 +#define _ITER_CHECK_RANGE_r02 1235 +#define _ITER_CHECK_RANGE_r12 1236 +#define _ITER_CHECK_RANGE_r22 1237 +#define _ITER_CHECK_RANGE_r33 1238 +#define _ITER_CHECK_TUPLE_r02 1239 +#define _ITER_CHECK_TUPLE_r12 1240 +#define _ITER_CHECK_TUPLE_r22 1241 +#define _ITER_CHECK_TUPLE_r33 1242 +#define _ITER_JUMP_LIST_r02 1243 +#define _ITER_JUMP_LIST_r12 1244 +#define _ITER_JUMP_LIST_r22 1245 +#define _ITER_JUMP_LIST_r33 1246 +#define _ITER_JUMP_RANGE_r02 1247 +#define _ITER_JUMP_RANGE_r12 1248 +#define _ITER_JUMP_RANGE_r22 1249 +#define _ITER_JUMP_RANGE_r33 1250 +#define _ITER_JUMP_TUPLE_r02 1251 +#define _ITER_JUMP_TUPLE_r12 1252 +#define _ITER_JUMP_TUPLE_r22 1253 +#define _ITER_JUMP_TUPLE_r33 1254 +#define _ITER_NEXT_LIST_r23 1255 +#define _ITER_NEXT_LIST_TIER_TWO_r23 1256 +#define _ITER_NEXT_RANGE_r03 1257 +#define _ITER_NEXT_RANGE_r13 1258 +#define _ITER_NEXT_RANGE_r23 1259 +#define _ITER_NEXT_TUPLE_r03 1260 +#define _ITER_NEXT_TUPLE_r13 1261 +#define _ITER_NEXT_TUPLE_r23 1262 +#define _JUMP_BACKWARD_NO_INTERRUPT_r00 1263 +#define _JUMP_BACKWARD_NO_INTERRUPT_r11 1264 +#define _JUMP_BACKWARD_NO_INTERRUPT_r22 1265 +#define _JUMP_BACKWARD_NO_INTERRUPT_r33 1266 +#define _JUMP_TO_TOP_r00 1267 +#define _LIST_APPEND_r10 1268 +#define _LIST_EXTEND_r11 1269 +#define _LOAD_ATTR_r10 1270 +#define _LOAD_ATTR_CLASS_r11 1271 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11 1272 +#define _LOAD_ATTR_INSTANCE_VALUE_r02 1273 +#define _LOAD_ATTR_INSTANCE_VALUE_r12 1274 +#define _LOAD_ATTR_INSTANCE_VALUE_r23 1275 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 1276 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 1277 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 1278 +#define _LOAD_ATTR_METHOD_NO_DICT_r02 1279 +#define _LOAD_ATTR_METHOD_NO_DICT_r12 1280 +#define _LOAD_ATTR_METHOD_NO_DICT_r23 1281 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 1282 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1283 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1284 +#define _LOAD_ATTR_MODULE_r12 1285 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1286 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1287 +#define _LOAD_ATTR_PROPERTY_FRAME_r01 1288 +#define _LOAD_ATTR_PROPERTY_FRAME_r11 1289 +#define _LOAD_ATTR_PROPERTY_FRAME_r22 1290 +#define _LOAD_ATTR_PROPERTY_FRAME_r33 1291 +#define _LOAD_ATTR_SLOT_r02 1292 +#define _LOAD_ATTR_SLOT_r12 1293 +#define _LOAD_ATTR_SLOT_r23 1294 +#define _LOAD_ATTR_WITH_HINT_r12 1295 +#define _LOAD_BUILD_CLASS_r01 1296 +#define _LOAD_BYTECODE_r00 1297 +#define _LOAD_COMMON_CONSTANT_r01 1298 +#define _LOAD_COMMON_CONSTANT_r12 1299 +#define _LOAD_COMMON_CONSTANT_r23 1300 +#define _LOAD_CONST_r01 1301 +#define _LOAD_CONST_r12 1302 +#define _LOAD_CONST_r23 1303 +#define _LOAD_CONST_INLINE_r01 1304 +#define _LOAD_CONST_INLINE_r12 1305 +#define _LOAD_CONST_INLINE_r23 1306 +#define _LOAD_CONST_INLINE_BORROW_r01 1307 +#define _LOAD_CONST_INLINE_BORROW_r12 1308 +#define _LOAD_CONST_INLINE_BORROW_r23 1309 +#define _LOAD_DEREF_r01 1310 +#define _LOAD_FAST_r01 1311 +#define _LOAD_FAST_r12 1312 +#define _LOAD_FAST_r23 1313 +#define _LOAD_FAST_0_r01 1314 +#define _LOAD_FAST_0_r12 1315 +#define _LOAD_FAST_0_r23 1316 +#define _LOAD_FAST_1_r01 1317 +#define _LOAD_FAST_1_r12 1318 +#define _LOAD_FAST_1_r23 1319 +#define _LOAD_FAST_2_r01 1320 +#define _LOAD_FAST_2_r12 1321 +#define _LOAD_FAST_2_r23 1322 +#define _LOAD_FAST_3_r01 1323 +#define _LOAD_FAST_3_r12 1324 +#define _LOAD_FAST_3_r23 1325 +#define _LOAD_FAST_4_r01 1326 +#define _LOAD_FAST_4_r12 1327 +#define _LOAD_FAST_4_r23 1328 +#define _LOAD_FAST_5_r01 1329 +#define _LOAD_FAST_5_r12 1330 +#define _LOAD_FAST_5_r23 1331 +#define _LOAD_FAST_6_r01 1332 +#define _LOAD_FAST_6_r12 1333 +#define _LOAD_FAST_6_r23 1334 +#define _LOAD_FAST_7_r01 1335 +#define _LOAD_FAST_7_r12 1336 +#define _LOAD_FAST_7_r23 1337 +#define _LOAD_FAST_AND_CLEAR_r01 1338 +#define _LOAD_FAST_AND_CLEAR_r12 1339 +#define _LOAD_FAST_AND_CLEAR_r23 1340 +#define _LOAD_FAST_BORROW_r01 1341 +#define _LOAD_FAST_BORROW_r12 1342 +#define _LOAD_FAST_BORROW_r23 1343 +#define _LOAD_FAST_BORROW_0_r01 1344 +#define _LOAD_FAST_BORROW_0_r12 1345 +#define _LOAD_FAST_BORROW_0_r23 1346 +#define _LOAD_FAST_BORROW_1_r01 1347 +#define _LOAD_FAST_BORROW_1_r12 1348 +#define _LOAD_FAST_BORROW_1_r23 1349 +#define _LOAD_FAST_BORROW_2_r01 1350 +#define _LOAD_FAST_BORROW_2_r12 1351 +#define _LOAD_FAST_BORROW_2_r23 1352 +#define _LOAD_FAST_BORROW_3_r01 1353 +#define _LOAD_FAST_BORROW_3_r12 1354 +#define _LOAD_FAST_BORROW_3_r23 1355 +#define _LOAD_FAST_BORROW_4_r01 1356 +#define _LOAD_FAST_BORROW_4_r12 1357 +#define _LOAD_FAST_BORROW_4_r23 1358 +#define _LOAD_FAST_BORROW_5_r01 1359 +#define _LOAD_FAST_BORROW_5_r12 1360 +#define _LOAD_FAST_BORROW_5_r23 1361 +#define _LOAD_FAST_BORROW_6_r01 1362 +#define _LOAD_FAST_BORROW_6_r12 1363 +#define _LOAD_FAST_BORROW_6_r23 1364 +#define _LOAD_FAST_BORROW_7_r01 1365 +#define _LOAD_FAST_BORROW_7_r12 1366 +#define _LOAD_FAST_BORROW_7_r23 1367 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1368 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1369 +#define _LOAD_FAST_CHECK_r01 1370 +#define _LOAD_FAST_CHECK_r12 1371 +#define _LOAD_FAST_CHECK_r23 1372 +#define _LOAD_FAST_LOAD_FAST_r02 1373 +#define _LOAD_FAST_LOAD_FAST_r13 1374 +#define _LOAD_FROM_DICT_OR_DEREF_r11 1375 +#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1376 +#define _LOAD_GLOBAL_r00 1377 +#define _LOAD_GLOBAL_BUILTINS_r01 1378 +#define _LOAD_GLOBAL_MODULE_r01 1379 +#define _LOAD_LOCALS_r01 1380 +#define _LOAD_LOCALS_r12 1381 +#define _LOAD_LOCALS_r23 1382 +#define _LOAD_NAME_r01 1383 +#define _LOAD_SMALL_INT_r01 1384 +#define _LOAD_SMALL_INT_r12 1385 +#define _LOAD_SMALL_INT_r23 1386 +#define _LOAD_SMALL_INT_0_r01 1387 +#define _LOAD_SMALL_INT_0_r12 1388 +#define _LOAD_SMALL_INT_0_r23 1389 +#define _LOAD_SMALL_INT_1_r01 1390 +#define _LOAD_SMALL_INT_1_r12 1391 +#define _LOAD_SMALL_INT_1_r23 1392 +#define _LOAD_SMALL_INT_2_r01 1393 +#define _LOAD_SMALL_INT_2_r12 1394 +#define _LOAD_SMALL_INT_2_r23 1395 +#define _LOAD_SMALL_INT_3_r01 1396 +#define _LOAD_SMALL_INT_3_r12 1397 +#define _LOAD_SMALL_INT_3_r23 1398 +#define _LOAD_SPECIAL_r00 1399 +#define _LOAD_SUPER_ATTR_ATTR_r31 1400 +#define _LOAD_SUPER_ATTR_METHOD_r32 1401 +#define _LOCK_OBJECT_r01 1402 +#define _LOCK_OBJECT_r11 1403 +#define _LOCK_OBJECT_r22 1404 +#define _LOCK_OBJECT_r33 1405 +#define _MAKE_CALLARGS_A_TUPLE_r33 1406 +#define _MAKE_CELL_r00 1407 +#define _MAKE_FUNCTION_r12 1408 +#define _MAKE_HEAP_SAFE_r01 1409 +#define _MAKE_HEAP_SAFE_r11 1410 +#define _MAKE_HEAP_SAFE_r22 1411 +#define _MAKE_HEAP_SAFE_r33 1412 +#define _MAKE_WARM_r00 1413 +#define _MAKE_WARM_r11 1414 +#define _MAKE_WARM_r22 1415 +#define _MAKE_WARM_r33 1416 +#define _MAP_ADD_r20 1417 +#define _MATCH_CLASS_r33 1418 +#define _MATCH_KEYS_r23 1419 +#define _MATCH_MAPPING_r02 1420 +#define _MATCH_MAPPING_r12 1421 +#define _MATCH_MAPPING_r23 1422 +#define _MATCH_SEQUENCE_r02 1423 +#define _MATCH_SEQUENCE_r12 1424 +#define _MATCH_SEQUENCE_r23 1425 +#define _MAYBE_EXPAND_METHOD_r00 1426 +#define _MAYBE_EXPAND_METHOD_KW_r11 1427 +#define _MONITOR_CALL_r00 1428 +#define _MONITOR_CALL_KW_r11 1429 +#define _MONITOR_JUMP_BACKWARD_r00 1430 +#define _MONITOR_JUMP_BACKWARD_r11 1431 +#define _MONITOR_JUMP_BACKWARD_r22 1432 +#define _MONITOR_JUMP_BACKWARD_r33 1433 +#define _MONITOR_RESUME_r00 1434 +#define _NOP_r00 1435 +#define _NOP_r11 1436 +#define _NOP_r22 1437 +#define _NOP_r33 1438 +#define _POP_EXCEPT_r10 1439 +#define _POP_ITER_r20 1440 +#define _POP_JUMP_IF_FALSE_r00 1441 +#define _POP_JUMP_IF_FALSE_r10 1442 +#define _POP_JUMP_IF_FALSE_r21 1443 +#define _POP_JUMP_IF_FALSE_r32 1444 +#define _POP_JUMP_IF_TRUE_r00 1445 +#define _POP_JUMP_IF_TRUE_r10 1446 +#define _POP_JUMP_IF_TRUE_r21 1447 +#define _POP_JUMP_IF_TRUE_r32 1448 +#define _POP_TOP_r10 1449 +#define _POP_TOP_FLOAT_r00 1450 +#define _POP_TOP_FLOAT_r10 1451 +#define _POP_TOP_FLOAT_r21 1452 +#define _POP_TOP_FLOAT_r32 1453 +#define _POP_TOP_INT_r00 1454 +#define _POP_TOP_INT_r10 1455 +#define _POP_TOP_INT_r21 1456 +#define _POP_TOP_INT_r32 1457 +#define _POP_TOP_NOP_r00 1458 +#define _POP_TOP_NOP_r10 1459 +#define _POP_TOP_NOP_r21 1460 +#define _POP_TOP_NOP_r32 1461 +#define _POP_TOP_OPARG_r00 1462 +#define _POP_TOP_UNICODE_r00 1463 +#define _POP_TOP_UNICODE_r10 1464 +#define _POP_TOP_UNICODE_r21 1465 +#define _POP_TOP_UNICODE_r32 1466 +#define _PUSH_EXC_INFO_r02 1467 +#define _PUSH_EXC_INFO_r12 1468 +#define _PUSH_EXC_INFO_r23 1469 +#define _PUSH_FRAME_r10 1470 +#define _PUSH_NULL_r01 1471 +#define _PUSH_NULL_r12 1472 +#define _PUSH_NULL_r23 1473 +#define _PUSH_NULL_CONDITIONAL_r00 1474 +#define _PUSH_TAGGED_ZERO_r01 1475 +#define _PUSH_TAGGED_ZERO_r12 1476 +#define _PUSH_TAGGED_ZERO_r23 1477 +#define _PY_FRAME_EX_r31 1478 +#define _PY_FRAME_GENERAL_r01 1479 +#define _PY_FRAME_KW_r11 1480 +#define _REPLACE_WITH_TRUE_r02 1481 +#define _REPLACE_WITH_TRUE_r12 1482 +#define _REPLACE_WITH_TRUE_r23 1483 +#define _RESUME_CHECK_r00 1484 +#define _RESUME_CHECK_r11 1485 +#define _RESUME_CHECK_r22 1486 +#define _RESUME_CHECK_r33 1487 +#define _RETURN_GENERATOR_r01 1488 +#define _RETURN_VALUE_r11 1489 +#define _RROT_3_r03 1490 +#define _RROT_3_r13 1491 +#define _RROT_3_r23 1492 +#define _RROT_3_r33 1493 +#define _SAVE_RETURN_OFFSET_r00 1494 +#define _SAVE_RETURN_OFFSET_r11 1495 +#define _SAVE_RETURN_OFFSET_r22 1496 +#define _SAVE_RETURN_OFFSET_r33 1497 +#define _SEND_r33 1498 +#define _SEND_GEN_FRAME_r33 1499 +#define _SETUP_ANNOTATIONS_r00 1500 +#define _SET_ADD_r10 1501 +#define _SET_FUNCTION_ATTRIBUTE_r01 1502 +#define _SET_FUNCTION_ATTRIBUTE_r11 1503 +#define _SET_FUNCTION_ATTRIBUTE_r21 1504 +#define _SET_FUNCTION_ATTRIBUTE_r32 1505 +#define _SET_IP_r00 1506 +#define _SET_IP_r11 1507 +#define _SET_IP_r22 1508 +#define _SET_IP_r33 1509 +#define _SET_UPDATE_r11 1510 +#define _SPILL_OR_RELOAD_r01 1511 +#define _SPILL_OR_RELOAD_r02 1512 +#define _SPILL_OR_RELOAD_r03 1513 +#define _SPILL_OR_RELOAD_r10 1514 +#define _SPILL_OR_RELOAD_r12 1515 +#define _SPILL_OR_RELOAD_r13 1516 +#define _SPILL_OR_RELOAD_r20 1517 +#define _SPILL_OR_RELOAD_r21 1518 +#define _SPILL_OR_RELOAD_r23 1519 +#define _SPILL_OR_RELOAD_r30 1520 +#define _SPILL_OR_RELOAD_r31 1521 +#define _SPILL_OR_RELOAD_r32 1522 +#define _START_EXECUTOR_r00 1523 +#define _STORE_ATTR_r20 1524 +#define _STORE_ATTR_INSTANCE_VALUE_r21 1525 +#define _STORE_ATTR_SLOT_r21 1526 +#define _STORE_ATTR_WITH_HINT_r21 1527 +#define _STORE_DEREF_r10 1528 +#define _STORE_FAST_LOAD_FAST_r11 1529 +#define _STORE_FAST_STORE_FAST_r20 1530 +#define _STORE_GLOBAL_r10 1531 +#define _STORE_NAME_r10 1532 +#define _STORE_SLICE_r30 1533 +#define _STORE_SUBSCR_r30 1534 +#define _STORE_SUBSCR_DICT_r31 1535 +#define _STORE_SUBSCR_DICT_KNOWN_HASH_r31 1536 +#define _STORE_SUBSCR_LIST_INT_r32 1537 +#define _SWAP_r11 1538 +#define _SWAP_2_r02 1539 +#define _SWAP_2_r12 1540 +#define _SWAP_2_r22 1541 +#define _SWAP_2_r33 1542 +#define _SWAP_3_r03 1543 +#define _SWAP_3_r13 1544 +#define _SWAP_3_r23 1545 +#define _SWAP_3_r33 1546 +#define _SWAP_FAST_r01 1547 +#define _SWAP_FAST_r11 1548 +#define _SWAP_FAST_r22 1549 +#define _SWAP_FAST_r33 1550 +#define _SWAP_FAST_0_r01 1551 +#define _SWAP_FAST_0_r11 1552 +#define _SWAP_FAST_0_r22 1553 +#define _SWAP_FAST_0_r33 1554 +#define _SWAP_FAST_1_r01 1555 +#define _SWAP_FAST_1_r11 1556 +#define _SWAP_FAST_1_r22 1557 +#define _SWAP_FAST_1_r33 1558 +#define _SWAP_FAST_2_r01 1559 +#define _SWAP_FAST_2_r11 1560 +#define _SWAP_FAST_2_r22 1561 +#define _SWAP_FAST_2_r33 1562 +#define _SWAP_FAST_3_r01 1563 +#define _SWAP_FAST_3_r11 1564 +#define _SWAP_FAST_3_r22 1565 +#define _SWAP_FAST_3_r33 1566 +#define _SWAP_FAST_4_r01 1567 +#define _SWAP_FAST_4_r11 1568 +#define _SWAP_FAST_4_r22 1569 +#define _SWAP_FAST_4_r33 1570 +#define _SWAP_FAST_5_r01 1571 +#define _SWAP_FAST_5_r11 1572 +#define _SWAP_FAST_5_r22 1573 +#define _SWAP_FAST_5_r33 1574 +#define _SWAP_FAST_6_r01 1575 +#define _SWAP_FAST_6_r11 1576 +#define _SWAP_FAST_6_r22 1577 +#define _SWAP_FAST_6_r33 1578 +#define _SWAP_FAST_7_r01 1579 +#define _SWAP_FAST_7_r11 1580 +#define _SWAP_FAST_7_r22 1581 +#define _SWAP_FAST_7_r33 1582 +#define _TIER2_RESUME_CHECK_r00 1583 +#define _TIER2_RESUME_CHECK_r11 1584 +#define _TIER2_RESUME_CHECK_r22 1585 +#define _TIER2_RESUME_CHECK_r33 1586 +#define _TO_BOOL_r11 1587 +#define _TO_BOOL_BOOL_r01 1588 +#define _TO_BOOL_BOOL_r11 1589 +#define _TO_BOOL_BOOL_r22 1590 +#define _TO_BOOL_BOOL_r33 1591 +#define _TO_BOOL_INT_r02 1592 +#define _TO_BOOL_INT_r12 1593 +#define _TO_BOOL_INT_r23 1594 +#define _TO_BOOL_LIST_r02 1595 +#define _TO_BOOL_LIST_r12 1596 +#define _TO_BOOL_LIST_r23 1597 +#define _TO_BOOL_NONE_r01 1598 +#define _TO_BOOL_NONE_r11 1599 +#define _TO_BOOL_NONE_r22 1600 +#define _TO_BOOL_NONE_r33 1601 +#define _TO_BOOL_STR_r02 1602 +#define _TO_BOOL_STR_r12 1603 +#define _TO_BOOL_STR_r23 1604 +#define _TRACE_RECORD_r00 1605 +#define _UNARY_INVERT_r12 1606 +#define _UNARY_NEGATIVE_r12 1607 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r02 1608 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r12 1609 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r23 1610 +#define _UNARY_NOT_r01 1611 +#define _UNARY_NOT_r11 1612 +#define _UNARY_NOT_r22 1613 +#define _UNARY_NOT_r33 1614 +#define _UNPACK_EX_r10 1615 +#define _UNPACK_SEQUENCE_r10 1616 +#define _UNPACK_SEQUENCE_LIST_r10 1617 +#define _UNPACK_SEQUENCE_TUPLE_r10 1618 +#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1619 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03 1620 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13 1621 +#define _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10 1622 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02 1623 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12 1624 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23 1625 +#define _WITH_EXCEPT_START_r33 1626 +#define _YIELD_VALUE_r11 1627 +#define MAX_UOP_REGS_ID 1627 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index bf361233560..8f543dbeeb8 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -11,15 +11,30 @@ extern "C" { #include #include "pycore_uop_ids.h" -extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1]; +#define MAX_CACHED_REGISTER 3 +extern const uint32_t _PyUop_Flags[MAX_UOP_ID+1]; typedef struct _rep_range { uint8_t start; uint8_t stop; } ReplicationRange; extern const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1]; -extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; +extern const char * const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1]; extern int _PyUop_num_popped(int opcode, int oparg); +typedef struct _pyuop_tos_cache_entry { + /* input depth is implicit in position */ + int8_t output; + int8_t exit; + int16_t opcode; +} _PyUopTOSentry; +typedef struct _PyUopCachingInfo { + uint8_t best[MAX_CACHED_REGISTER + 1]; + _PyUopTOSentry entries[MAX_CACHED_REGISTER + 1]; +} _PyUopCachingInfo; +extern const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1]; +extern const uint16_t _PyUop_SpillsAndReloads[4][4]; +extern const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1]; + #ifdef NEED_OPCODE_METADATA -const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { +const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_NOP] = HAS_PURE_FLAG, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -44,50 +59,48 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_FAST_BORROW_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_BORROW] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_LOAD_FAST_BORROW_LOAD_FAST_BORROW] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG, [_LOAD_SMALL_INT_0] = 0, [_LOAD_SMALL_INT_1] = 0, [_LOAD_SMALL_INT_2] = 0, [_LOAD_SMALL_INT_3] = 0, [_LOAD_SMALL_INT] = HAS_ARG_FLAG, - [_STORE_FAST_0] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_1] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_2] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_3] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_4] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_5] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_6] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_7] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_SWAP_FAST_0] = HAS_LOCAL_FLAG, + [_SWAP_FAST_1] = HAS_LOCAL_FLAG, + [_SWAP_FAST_2] = HAS_LOCAL_FLAG, + [_SWAP_FAST_3] = HAS_LOCAL_FLAG, + [_SWAP_FAST_4] = HAS_LOCAL_FLAG, + [_SWAP_FAST_5] = HAS_LOCAL_FLAG, + [_SWAP_FAST_6] = HAS_LOCAL_FLAG, + [_SWAP_FAST_7] = HAS_LOCAL_FLAG, + [_SWAP_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_POP_TOP] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_POP_TOP_NOP] = 0, [_POP_TOP_INT] = 0, [_POP_TOP_FLOAT] = 0, [_POP_TOP_UNICODE] = 0, - [_POP_TWO] = HAS_ESCAPES_FLAG, + [_POP_TOP_OPARG] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_PUSH_NULL] = HAS_PURE_FLAG, [_END_FOR] = HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG, [_POP_ITER] = HAS_ESCAPES_FLAG, [_END_SEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_UNARY_NOT] = HAS_PURE_FLAG, + [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_UNARY_NEGATIVE_FLOAT_INPLACE] = 0, + [_UNARY_NOT] = 0, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_TO_BOOL_BOOL] = HAS_EXIT_FLAG, - [_TO_BOOL_INT] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_TO_BOOL_INT] = 0, [_GUARD_NOS_LIST] = HAS_EXIT_FLAG, [_GUARD_TOS_LIST] = HAS_EXIT_FLAG, [_GUARD_TOS_SLICE] = HAS_EXIT_FLAG, - [_TO_BOOL_LIST] = HAS_ESCAPES_FLAG, + [_TO_BOOL_LIST] = 0, [_TO_BOOL_NONE] = HAS_EXIT_FLAG, + [_GUARD_NOS_COMPACT_ASCII] = HAS_EXIT_FLAG, [_GUARD_NOS_UNICODE] = HAS_EXIT_FLAG, [_GUARD_TOS_UNICODE] = HAS_EXIT_FLAG, - [_TO_BOOL_STR] = HAS_ESCAPES_FLAG, - [_REPLACE_WITH_TRUE] = HAS_ESCAPES_FLAG, - [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_TO_BOOL_STR] = 0, + [_REPLACE_WITH_TRUE] = 0, + [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_INT] = HAS_EXIT_FLAG, [_GUARD_TOS_INT] = HAS_EXIT_FLAG, [_GUARD_NOS_OVERFLOWED] = HAS_EXIT_FLAG, @@ -95,53 +108,78 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_MULTIPLY_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_INT_INPLACE] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBTRACT_INT_INPLACE] = HAS_EXIT_FLAG, + [_BINARY_OP_MULTIPLY_INT_INPLACE] = HAS_EXIT_FLAG, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT] = HAS_EXIT_FLAG, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT] = HAS_EXIT_FLAG, [_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG, [_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG, - [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_BINARY_OP_EXTEND] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_EXTEND] = HAS_ESCAPES_FLAG, + [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT_INPLACE] = 0, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE] = 0, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE] = 0, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT] = 0, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT] = 0, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT] = 0, + [_BINARY_OP_TRUEDIV_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_BINARY_OP_EXTEND_LHS] = HAS_EXIT_FLAG, + [_GUARD_BINARY_OP_EXTEND_RHS] = HAS_EXIT_FLAG, + [_GUARD_BINARY_OP_EXTEND] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_EXTEND] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_SUBSCR_LIST_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_SUBSCR_STR_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_LIST_INT] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_LIST_SLICE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_STR_INT] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBSCR_USTR_INT] = HAS_EXIT_FLAG, [_GUARD_NOS_TUPLE] = HAS_EXIT_FLAG, [_GUARD_TOS_TUPLE] = HAS_EXIT_FLAG, - [_BINARY_OP_SUBSCR_TUPLE_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBSCR_TUPLE_INT] = 0, [_GUARD_NOS_DICT] = HAS_EXIT_FLAG, + [_GUARD_NOS_ANY_DICT] = HAS_EXIT_FLAG, + [_GUARD_TOS_ANY_DICT] = HAS_EXIT_FLAG, [_GUARD_TOS_DICT] = HAS_EXIT_FLAG, - [_BINARY_OP_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_SUBSCR_CHECK_FUNC] = HAS_DEOPT_FLAG, + [_GUARD_TOS_FROZENDICT] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_CHECK_FUNC] = HAS_EXIT_FLAG, [_BINARY_OP_SUBSCR_INIT_CALL] = 0, [_LIST_APPEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SUBSCR_DICT_KNOWN_HASH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_RETURN_VALUE] = HAS_ESCAPES_FLAG, + [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_HEAP_SAFE] = 0, + [_RETURN_VALUE] = HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_YIELD_VALUE] = HAS_ARG_FLAG, + [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE] = 0, + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE] = 0, + [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_UNPACK_SEQUENCE_UNIQUE_TUPLE] = HAS_ARG_FLAG, [_UNPACK_SEQUENCE_LIST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_EX] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -167,53 +205,65 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BUILD_TEMPLATE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_TUPLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, [_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_SET_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_SET_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_BUILD_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SETUP_ANNOTATIONS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DICT_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DICT_MERGE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DICT_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_DICT_MERGE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAP_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_NOS_TYPE_VERSION] = HAS_EXIT_FLAG, + [_GUARD_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_TYPE_VERSION] = HAS_EXIT_FLAG, - [_GUARD_TYPE_VERSION_AND_LOCK] = HAS_EXIT_FLAG, - [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG, - [_LOAD_ATTR_INSTANCE_VALUE] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_ATTR_SLOT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_TYPE_VERSION_LOCKED] = HAS_EXIT_FLAG, + [_GUARD_TYPE] = HAS_EXIT_FLAG, + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_EXIT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_MODULE] = HAS_EXIT_FLAG, + [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG, + [_LOAD_ATTR_SLOT] = HAS_EXIT_FLAG, [_CHECK_ATTR_CLASS] = HAS_EXIT_FLAG, [_LOAD_ATTR_CLASS] = HAS_ESCAPES_FLAG, - [_LOAD_ATTR_PROPERTY_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_ATTR_PROPERTY_FRAME] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG, [_GUARD_DORV_NO_DICT] = HAS_EXIT_FLAG, [_STORE_ATTR_INSTANCE_VALUE] = HAS_ESCAPES_FLAG, + [_LOCK_OBJECT] = HAS_DEOPT_FLAG, [_STORE_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_STORE_ATTR_SLOT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG, [_COMPARE_OP_INT] = HAS_ARG_FLAG, [_COMPARE_OP_STR] = HAS_ARG_FLAG, - [_IS_OP] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_TOS_ANY_SET] = HAS_DEOPT_FLAG, - [_CONTAINS_OP_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CONTAINS_OP_DICT] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_IS_OP] = HAS_ARG_FLAG, + [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_TOS_ANY_SET] = HAS_EXIT_FLAG, + [_GUARD_TOS_SET] = HAS_EXIT_FLAG, + [_GUARD_TOS_FROZENSET] = HAS_EXIT_FLAG, + [_CONTAINS_OP_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CONTAINS_OP_DICT] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EG_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EXC_MATCH] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_IMPORT_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_IMPORT_FROM] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_IS_NONE] = HAS_ESCAPES_FLAG, [_GET_LEN] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_MATCH_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MATCH_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MATCH_MAPPING] = 0, [_MATCH_SEQUENCE] = 0, [_MATCH_KEYS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GET_ITER] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_ITERATOR] = HAS_EXIT_FLAG, + [_GUARD_ITER_VIRTUAL] = HAS_EXIT_FLAG, + [_PUSH_TAGGED_ZERO] = 0, + [_GET_ITER_TRAD] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_NOS_ITER_VIRTUAL] = HAS_EXIT_FLAG, + [_FOR_ITER_VIRTUAL_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, [_ITER_NEXT_LIST_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, @@ -223,21 +273,21 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_ITER_CHECK_RANGE] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_EXIT_FLAG, [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG, - [_FOR_ITER_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_FOR_ITER_GEN_FRAME] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_INSERT_NULL] = 0, [_LOAD_SPECIAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_EXC_INFO] = 0, - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG, - [_GUARD_KEYS_VERSION] = HAS_DEOPT_FLAG, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_EXIT_FLAG, + [_GUARD_KEYS_VERSION] = HAS_EXIT_FLAG, [_LOAD_ATTR_METHOD_WITH_VALUES] = HAS_ARG_FLAG, [_LOAD_ATTR_METHOD_NO_DICT] = HAS_ARG_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG, + [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_EXIT_FLAG, [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, [_MAYBE_EXPAND_METHOD] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_CHECK_FUNCTION_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CHECK_FUNCTION_VERSION_INLINE] = HAS_EXIT_FLAG, [_CHECK_METHOD_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -256,44 +306,62 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_PUSH_FRAME] = 0, - [_GUARD_NOS_NULL] = HAS_DEOPT_FLAG, + [_PUSH_FRAME] = HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_GUARD_NOS_NULL] = HAS_EXIT_FLAG, [_GUARD_NOS_NOT_NULL] = HAS_EXIT_FLAG, - [_GUARD_THIRD_NULL] = HAS_DEOPT_FLAG, - [_GUARD_CALLABLE_TYPE_1] = HAS_DEOPT_FLAG, - [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_CALLABLE_STR_1] = HAS_DEOPT_FLAG, - [_CALL_STR_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_CALLABLE_TUPLE_1] = HAS_DEOPT_FLAG, - [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CHECK_AND_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_THIRD_NULL] = HAS_EXIT_FLAG, + [_GUARD_CALLABLE_TYPE_1] = HAS_EXIT_FLAG, + [_CALL_TYPE_1] = HAS_ARG_FLAG, + [_GUARD_CALLABLE_STR_1] = HAS_EXIT_FLAG, + [_CALL_STR_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_TUPLE_1] = HAS_EXIT_FLAG, + [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_OBJECT] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_CALLABLE_LEN] = HAS_DEOPT_FLAG, + [_GUARD_CALLABLE_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_BUILTIN_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_LEN] = HAS_EXIT_FLAG, [_CALL_LEN] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_CALLABLE_ISINSTANCE] = HAS_DEOPT_FLAG, + [_GUARD_CALLABLE_ISINSTANCE] = HAS_EXIT_FLAG, [_CALL_ISINSTANCE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_CALLABLE_LIST_APPEND] = HAS_DEOPT_FLAG, - [_CALL_LIST_APPEND] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_LIST_APPEND] = HAS_EXIT_FLAG, + [_CALL_LIST_APPEND] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_RECURSION_LIMIT] = HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_O_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAYBE_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CHECK_METHOD_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_CHECK_IS_NOT_PY_CALLABLE_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CALL_KW_NON_PY] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAKE_CALLARGS_A_TUPLE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_IS_PY_CALLABLE_EX] = HAS_EXIT_FLAG, + [_PY_FRAME_EX] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, + [_CHECK_IS_NOT_PY_CALLABLE_EX] = HAS_EXIT_FLAG, + [_CALL_FUNCTION_EX_NON_PY_GENERAL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG, - [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -308,6 +376,16 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_SET_POP_4] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_SET_POP_5] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_SET_POP_6] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_SET_POP_7] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_SET_POP] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_GUARD_BIT_IS_UNSET_POP_4] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_UNSET_POP_5] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_UNSET_POP_6] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_UNSET_POP_7] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_UNSET_POP] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, [_JUMP_TO_TOP] = 0, @@ -315,360 +393,5712 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, [_EXIT_TRACE] = HAS_ESCAPES_FLAG, + [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, - [_POP_TOP_LOAD_CONST_INLINE] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, - [_POP_CALL] = HAS_ESCAPES_FLAG, - [_POP_CALL_ONE] = HAS_ESCAPES_FLAG, - [_POP_CALL_TWO] = HAS_ESCAPES_FLAG, - [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, - [_POP_TWO_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, - [_POP_CALL_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, - [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, - [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, - [_LOAD_CONST_UNDER_INLINE] = 0, - [_LOAD_CONST_UNDER_INLINE_BORROW] = 0, - [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, + [_RROT_3] = HAS_PURE_FLAG, [_START_EXECUTOR] = HAS_DEOPT_FLAG, [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, - [_DEOPT] = 0, - [_HANDLE_PENDING_AND_DEOPT] = HAS_ESCAPES_FLAG, - [_ERROR_POP_N] = HAS_ARG_FLAG, + [_DEOPT] = HAS_SYNC_SP_FLAG, + [_HANDLE_PENDING_AND_DEOPT] = HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, + [_ERROR_POP_N] = HAS_ARG_FLAG | HAS_SYNC_SP_FLAG, + [_SPILL_OR_RELOAD] = 0, [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, - [_COLD_EXIT] = HAS_ESCAPES_FLAG, + [_COLD_EXIT] = HAS_SYNC_SP_FLAG, + [_COLD_DYNAMIC_EXIT] = HAS_SYNC_SP_FLAG, + [_GUARD_CODE_VERSION__PUSH_FRAME] = HAS_EXIT_FLAG, + [_GUARD_CODE_VERSION_YIELD_VALUE] = HAS_EXIT_FLAG, + [_GUARD_CODE_VERSION_RETURN_VALUE] = HAS_EXIT_FLAG, + [_GUARD_CODE_VERSION_RETURN_GENERATOR] = HAS_EXIT_FLAG, + [_GUARD_IP__PUSH_FRAME] = HAS_EXIT_FLAG, + [_GUARD_IP_YIELD_VALUE] = HAS_EXIT_FLAG, + [_GUARD_IP_RETURN_VALUE] = HAS_EXIT_FLAG, + [_GUARD_IP_RETURN_GENERATOR] = HAS_EXIT_FLAG, + [_RECORD_TOS] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_TOS_TYPE] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_NOS] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_NOS_TYPE] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_NOS_GEN_FUNC] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_3OS_GEN_FUNC] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_4OS] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_CALLABLE] = HAS_ARG_FLAG | HAS_RECORDS_VALUE_FLAG, + [_RECORD_CALLABLE_KW] = HAS_ARG_FLAG | HAS_RECORDS_VALUE_FLAG, + [_RECORD_BOUND_METHOD] = HAS_ARG_FLAG | HAS_RECORDS_VALUE_FLAG, + [_RECORD_CODE] = HAS_RECORDS_VALUE_FLAG, }; const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { [_LOAD_FAST] = { 0, 8 }, [_LOAD_FAST_BORROW] = { 0, 8 }, [_LOAD_SMALL_INT] = { 0, 4 }, - [_STORE_FAST] = { 0, 8 }, + [_SWAP_FAST] = { 0, 8 }, [_INIT_CALL_PY_EXACT_ARGS] = { 0, 5 }, [_COPY] = { 1, 4 }, [_SWAP] = { 2, 4 }, + [_GUARD_BIT_IS_SET_POP] = { 4, 8 }, + [_GUARD_BIT_IS_UNSET_POP] = { 4, 8 }, }; -const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { +const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { + [_NOP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _NOP_r00 }, + { 1, 1, _NOP_r11 }, + { 2, 2, _NOP_r22 }, + { 3, 3, _NOP_r33 }, + }, + }, + [_CHECK_PERIODIC] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_PERIODIC_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_RESUME_CHECK] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _RESUME_CHECK_r00 }, + { 1, 1, _RESUME_CHECK_r11 }, + { 2, 2, _RESUME_CHECK_r22 }, + { 3, 3, _RESUME_CHECK_r33 }, + }, + }, + [_LOAD_FAST_CHECK] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_CHECK_r01 }, + { 2, 1, _LOAD_FAST_CHECK_r12 }, + { 3, 2, _LOAD_FAST_CHECK_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_0] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_0_r01 }, + { 2, 1, _LOAD_FAST_0_r12 }, + { 3, 2, _LOAD_FAST_0_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_1] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_1_r01 }, + { 2, 1, _LOAD_FAST_1_r12 }, + { 3, 2, _LOAD_FAST_1_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_2] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_2_r01 }, + { 2, 1, _LOAD_FAST_2_r12 }, + { 3, 2, _LOAD_FAST_2_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_3] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_3_r01 }, + { 2, 1, _LOAD_FAST_3_r12 }, + { 3, 2, _LOAD_FAST_3_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_4] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_4_r01 }, + { 2, 1, _LOAD_FAST_4_r12 }, + { 3, 2, _LOAD_FAST_4_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_5] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_5_r01 }, + { 2, 1, _LOAD_FAST_5_r12 }, + { 3, 2, _LOAD_FAST_5_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_6] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_6_r01 }, + { 2, 1, _LOAD_FAST_6_r12 }, + { 3, 2, _LOAD_FAST_6_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_7] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_7_r01 }, + { 2, 1, _LOAD_FAST_7_r12 }, + { 3, 2, _LOAD_FAST_7_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_r01 }, + { 2, 1, _LOAD_FAST_r12 }, + { 3, 2, _LOAD_FAST_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_0] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_0_r01 }, + { 2, 1, _LOAD_FAST_BORROW_0_r12 }, + { 3, 2, _LOAD_FAST_BORROW_0_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_1] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_1_r01 }, + { 2, 1, _LOAD_FAST_BORROW_1_r12 }, + { 3, 2, _LOAD_FAST_BORROW_1_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_2] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_2_r01 }, + { 2, 1, _LOAD_FAST_BORROW_2_r12 }, + { 3, 2, _LOAD_FAST_BORROW_2_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_3] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_3_r01 }, + { 2, 1, _LOAD_FAST_BORROW_3_r12 }, + { 3, 2, _LOAD_FAST_BORROW_3_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_4] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_4_r01 }, + { 2, 1, _LOAD_FAST_BORROW_4_r12 }, + { 3, 2, _LOAD_FAST_BORROW_4_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_5] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_5_r01 }, + { 2, 1, _LOAD_FAST_BORROW_5_r12 }, + { 3, 2, _LOAD_FAST_BORROW_5_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_6] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_6_r01 }, + { 2, 1, _LOAD_FAST_BORROW_6_r12 }, + { 3, 2, _LOAD_FAST_BORROW_6_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_7] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_7_r01 }, + { 2, 1, _LOAD_FAST_BORROW_7_r12 }, + { 3, 2, _LOAD_FAST_BORROW_7_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_r01 }, + { 2, 1, _LOAD_FAST_BORROW_r12 }, + { 3, 2, _LOAD_FAST_BORROW_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_AND_CLEAR] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_AND_CLEAR_r01 }, + { 2, 1, _LOAD_FAST_AND_CLEAR_r12 }, + { 3, 2, _LOAD_FAST_AND_CLEAR_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_CONST] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_CONST_r01 }, + { 2, 1, _LOAD_CONST_r12 }, + { 3, 2, _LOAD_CONST_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT_0] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_0_r01 }, + { 2, 1, _LOAD_SMALL_INT_0_r12 }, + { 3, 2, _LOAD_SMALL_INT_0_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT_1] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_1_r01 }, + { 2, 1, _LOAD_SMALL_INT_1_r12 }, + { 3, 2, _LOAD_SMALL_INT_1_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT_2] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_2_r01 }, + { 2, 1, _LOAD_SMALL_INT_2_r12 }, + { 3, 2, _LOAD_SMALL_INT_2_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT_3] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_3_r01 }, + { 2, 1, _LOAD_SMALL_INT_3_r12 }, + { 3, 2, _LOAD_SMALL_INT_3_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_r01 }, + { 2, 1, _LOAD_SMALL_INT_r12 }, + { 3, 2, _LOAD_SMALL_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_SWAP_FAST_0] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_0_r01 }, + { 1, 1, _SWAP_FAST_0_r11 }, + { 2, 2, _SWAP_FAST_0_r22 }, + { 3, 3, _SWAP_FAST_0_r33 }, + }, + }, + [_SWAP_FAST_1] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_1_r01 }, + { 1, 1, _SWAP_FAST_1_r11 }, + { 2, 2, _SWAP_FAST_1_r22 }, + { 3, 3, _SWAP_FAST_1_r33 }, + }, + }, + [_SWAP_FAST_2] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_2_r01 }, + { 1, 1, _SWAP_FAST_2_r11 }, + { 2, 2, _SWAP_FAST_2_r22 }, + { 3, 3, _SWAP_FAST_2_r33 }, + }, + }, + [_SWAP_FAST_3] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_3_r01 }, + { 1, 1, _SWAP_FAST_3_r11 }, + { 2, 2, _SWAP_FAST_3_r22 }, + { 3, 3, _SWAP_FAST_3_r33 }, + }, + }, + [_SWAP_FAST_4] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_4_r01 }, + { 1, 1, _SWAP_FAST_4_r11 }, + { 2, 2, _SWAP_FAST_4_r22 }, + { 3, 3, _SWAP_FAST_4_r33 }, + }, + }, + [_SWAP_FAST_5] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_5_r01 }, + { 1, 1, _SWAP_FAST_5_r11 }, + { 2, 2, _SWAP_FAST_5_r22 }, + { 3, 3, _SWAP_FAST_5_r33 }, + }, + }, + [_SWAP_FAST_6] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_6_r01 }, + { 1, 1, _SWAP_FAST_6_r11 }, + { 2, 2, _SWAP_FAST_6_r22 }, + { 3, 3, _SWAP_FAST_6_r33 }, + }, + }, + [_SWAP_FAST_7] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_7_r01 }, + { 1, 1, _SWAP_FAST_7_r11 }, + { 2, 2, _SWAP_FAST_7_r22 }, + { 3, 3, _SWAP_FAST_7_r33 }, + }, + }, + [_SWAP_FAST] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_r01 }, + { 1, 1, _SWAP_FAST_r11 }, + { 2, 2, _SWAP_FAST_r22 }, + { 3, 3, _SWAP_FAST_r33 }, + }, + }, + [_POP_TOP] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _POP_TOP_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_POP_TOP_NOP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _POP_TOP_NOP_r00 }, + { 0, 1, _POP_TOP_NOP_r10 }, + { 1, 2, _POP_TOP_NOP_r21 }, + { 2, 3, _POP_TOP_NOP_r32 }, + }, + }, + [_POP_TOP_INT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _POP_TOP_INT_r00 }, + { 0, 1, _POP_TOP_INT_r10 }, + { 1, 2, _POP_TOP_INT_r21 }, + { 2, 3, _POP_TOP_INT_r32 }, + }, + }, + [_POP_TOP_FLOAT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _POP_TOP_FLOAT_r00 }, + { 0, 1, _POP_TOP_FLOAT_r10 }, + { 1, 2, _POP_TOP_FLOAT_r21 }, + { 2, 3, _POP_TOP_FLOAT_r32 }, + }, + }, + [_POP_TOP_UNICODE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _POP_TOP_UNICODE_r00 }, + { 0, 1, _POP_TOP_UNICODE_r10 }, + { 1, 2, _POP_TOP_UNICODE_r21 }, + { 2, 3, _POP_TOP_UNICODE_r32 }, + }, + }, + [_POP_TOP_OPARG] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _POP_TOP_OPARG_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_PUSH_NULL] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _PUSH_NULL_r01 }, + { 2, 1, _PUSH_NULL_r12 }, + { 3, 2, _PUSH_NULL_r23 }, + { -1, -1, -1 }, + }, + }, + [_END_FOR] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _END_FOR_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_POP_ITER] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 2, _POP_ITER_r20 }, + { -1, -1, -1 }, + }, + }, + [_END_SEND] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _END_SEND_r31 }, + }, + }, + [_UNARY_NEGATIVE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _UNARY_NEGATIVE_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNARY_NEGATIVE_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _UNARY_NEGATIVE_FLOAT_INPLACE_r02 }, + { 2, 1, _UNARY_NEGATIVE_FLOAT_INPLACE_r12 }, + { 3, 2, _UNARY_NEGATIVE_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_UNARY_NOT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _UNARY_NOT_r01 }, + { 1, 1, _UNARY_NOT_r11 }, + { 2, 2, _UNARY_NOT_r22 }, + { 3, 3, _UNARY_NOT_r33 }, + }, + }, + [_TO_BOOL] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _TO_BOOL_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_TO_BOOL_BOOL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _TO_BOOL_BOOL_r01 }, + { 1, 1, _TO_BOOL_BOOL_r11 }, + { 2, 2, _TO_BOOL_BOOL_r22 }, + { 3, 3, _TO_BOOL_BOOL_r33 }, + }, + }, + [_TO_BOOL_INT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _TO_BOOL_INT_r02 }, + { 2, 1, _TO_BOOL_INT_r12 }, + { 3, 2, _TO_BOOL_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_LIST] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_LIST_r02 }, + { 2, 1, _GUARD_NOS_LIST_r12 }, + { 2, 2, _GUARD_NOS_LIST_r22 }, + { 3, 3, _GUARD_NOS_LIST_r33 }, + }, + }, + [_GUARD_TOS_LIST] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_LIST_r01 }, + { 1, 1, _GUARD_TOS_LIST_r11 }, + { 2, 2, _GUARD_TOS_LIST_r22 }, + { 3, 3, _GUARD_TOS_LIST_r33 }, + }, + }, + [_GUARD_TOS_SLICE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_SLICE_r01 }, + { 1, 1, _GUARD_TOS_SLICE_r11 }, + { 2, 2, _GUARD_TOS_SLICE_r22 }, + { 3, 3, _GUARD_TOS_SLICE_r33 }, + }, + }, + [_TO_BOOL_LIST] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _TO_BOOL_LIST_r02 }, + { 2, 1, _TO_BOOL_LIST_r12 }, + { 3, 2, _TO_BOOL_LIST_r23 }, + { -1, -1, -1 }, + }, + }, + [_TO_BOOL_NONE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _TO_BOOL_NONE_r01 }, + { 1, 1, _TO_BOOL_NONE_r11 }, + { 2, 2, _TO_BOOL_NONE_r22 }, + { 3, 3, _TO_BOOL_NONE_r33 }, + }, + }, + [_GUARD_NOS_COMPACT_ASCII] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_COMPACT_ASCII_r02 }, + { 2, 1, _GUARD_NOS_COMPACT_ASCII_r12 }, + { 2, 2, _GUARD_NOS_COMPACT_ASCII_r22 }, + { 3, 3, _GUARD_NOS_COMPACT_ASCII_r33 }, + }, + }, + [_GUARD_NOS_UNICODE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_UNICODE_r02 }, + { 2, 1, _GUARD_NOS_UNICODE_r12 }, + { 2, 2, _GUARD_NOS_UNICODE_r22 }, + { 3, 3, _GUARD_NOS_UNICODE_r33 }, + }, + }, + [_GUARD_TOS_UNICODE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_UNICODE_r01 }, + { 1, 1, _GUARD_TOS_UNICODE_r11 }, + { 2, 2, _GUARD_TOS_UNICODE_r22 }, + { 3, 3, _GUARD_TOS_UNICODE_r33 }, + }, + }, + [_TO_BOOL_STR] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _TO_BOOL_STR_r02 }, + { 2, 1, _TO_BOOL_STR_r12 }, + { 3, 2, _TO_BOOL_STR_r23 }, + { -1, -1, -1 }, + }, + }, + [_REPLACE_WITH_TRUE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _REPLACE_WITH_TRUE_r02 }, + { 2, 1, _REPLACE_WITH_TRUE_r12 }, + { 3, 2, _REPLACE_WITH_TRUE_r23 }, + { -1, -1, -1 }, + }, + }, + [_UNARY_INVERT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _UNARY_INVERT_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_INT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_INT_r02 }, + { 2, 1, _GUARD_NOS_INT_r12 }, + { 2, 2, _GUARD_NOS_INT_r22 }, + { 3, 3, _GUARD_NOS_INT_r33 }, + }, + }, + [_GUARD_TOS_INT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_INT_r01 }, + { 1, 1, _GUARD_TOS_INT_r11 }, + { 2, 2, _GUARD_TOS_INT_r22 }, + { 3, 3, _GUARD_TOS_INT_r33 }, + }, + }, + [_GUARD_NOS_OVERFLOWED] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_OVERFLOWED_r02 }, + { 2, 1, _GUARD_NOS_OVERFLOWED_r12 }, + { 2, 2, _GUARD_NOS_OVERFLOWED_r22 }, + { 3, 3, _GUARD_NOS_OVERFLOWED_r33 }, + }, + }, + [_GUARD_TOS_OVERFLOWED] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_OVERFLOWED_r01 }, + { 1, 1, _GUARD_TOS_OVERFLOWED_r11 }, + { 2, 2, _GUARD_TOS_OVERFLOWED_r22 }, + { 3, 3, _GUARD_TOS_OVERFLOWED_r33 }, + }, + }, + [_BINARY_OP_MULTIPLY_INT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_INT_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_INT_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_INT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_INT_r03 }, + { 3, 1, _BINARY_OP_ADD_INT_r13 }, + { 3, 2, _BINARY_OP_ADD_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_INT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_INT_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_INT_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_INT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_INT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_ADD_INT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_ADD_INT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_INT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_INT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_INT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_INT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_MULTIPLY_INT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_INT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_INT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_INT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_INT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_ADD_INT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_ADD_INT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_FLOAT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_FLOAT_r02 }, + { 2, 1, _GUARD_NOS_FLOAT_r12 }, + { 2, 2, _GUARD_NOS_FLOAT_r22 }, + { 3, 3, _GUARD_NOS_FLOAT_r33 }, + }, + }, + [_GUARD_TOS_FLOAT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_FLOAT_r01 }, + { 1, 1, _GUARD_TOS_FLOAT_r11 }, + { 2, 2, _GUARD_TOS_FLOAT_r22 }, + { 3, 3, _GUARD_TOS_FLOAT_r33 }, + }, + }, + [_BINARY_OP_MULTIPLY_FLOAT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_FLOAT_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_FLOAT_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_FLOAT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_FLOAT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_FLOAT_r03 }, + { 3, 1, _BINARY_OP_ADD_FLOAT_r13 }, + { 3, 2, _BINARY_OP_ADD_FLOAT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_FLOAT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_FLOAT_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_FLOAT_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_FLOAT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_FLOAT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_ADD_FLOAT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_ADD_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_TRUEDIV_FLOAT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_TRUEDIV_FLOAT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_UNICODE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_UNICODE_r03 }, + { 3, 1, _BINARY_OP_ADD_UNICODE_r13 }, + { 3, 2, _BINARY_OP_ADD_UNICODE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_INPLACE_ADD_UNICODE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _BINARY_OP_INPLACE_ADD_UNICODE_r21 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_BINARY_OP_EXTEND_LHS] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_BINARY_OP_EXTEND_LHS_r02 }, + { 2, 1, _GUARD_BINARY_OP_EXTEND_LHS_r12 }, + { 2, 2, _GUARD_BINARY_OP_EXTEND_LHS_r22 }, + { 3, 3, _GUARD_BINARY_OP_EXTEND_LHS_r33 }, + }, + }, + [_GUARD_BINARY_OP_EXTEND_RHS] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_BINARY_OP_EXTEND_RHS_r02 }, + { 2, 1, _GUARD_BINARY_OP_EXTEND_RHS_r12 }, + { 2, 2, _GUARD_BINARY_OP_EXTEND_RHS_r22 }, + { 3, 3, _GUARD_BINARY_OP_EXTEND_RHS_r33 }, + }, + }, + [_GUARD_BINARY_OP_EXTEND] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 2, _GUARD_BINARY_OP_EXTEND_r22 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_EXTEND] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_EXTEND_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_SLICE] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _BINARY_SLICE_r31 }, + }, + }, + [_STORE_SLICE] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 3, _STORE_SLICE_r30 }, + }, + }, + [_BINARY_OP_SUBSCR_LIST_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_LIST_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_LIST_SLICE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_LIST_SLICE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_STR_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_STR_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_USTR_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_USTR_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_TUPLE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_TUPLE_r02 }, + { 2, 1, _GUARD_NOS_TUPLE_r12 }, + { 2, 2, _GUARD_NOS_TUPLE_r22 }, + { 3, 3, _GUARD_NOS_TUPLE_r33 }, + }, + }, + [_GUARD_TOS_TUPLE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_TUPLE_r01 }, + { 1, 1, _GUARD_TOS_TUPLE_r11 }, + { 2, 2, _GUARD_TOS_TUPLE_r22 }, + { 3, 3, _GUARD_TOS_TUPLE_r33 }, + }, + }, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02 }, + { 2, 1, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12 }, + { 2, 2, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22 }, + { 3, 3, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33 }, + }, + }, + [_BINARY_OP_SUBSCR_TUPLE_INT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBSCR_TUPLE_INT_r03 }, + { 3, 1, _BINARY_OP_SUBSCR_TUPLE_INT_r13 }, + { 3, 2, _BINARY_OP_SUBSCR_TUPLE_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_DICT_r02 }, + { 2, 1, _GUARD_NOS_DICT_r12 }, + { 2, 2, _GUARD_NOS_DICT_r22 }, + { 3, 3, _GUARD_NOS_DICT_r33 }, + }, + }, + [_GUARD_NOS_ANY_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_ANY_DICT_r02 }, + { 2, 1, _GUARD_NOS_ANY_DICT_r12 }, + { 2, 2, _GUARD_NOS_ANY_DICT_r22 }, + { 3, 3, _GUARD_NOS_ANY_DICT_r33 }, + }, + }, + [_GUARD_TOS_ANY_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_ANY_DICT_r01 }, + { 1, 1, _GUARD_TOS_ANY_DICT_r11 }, + { 2, 2, _GUARD_TOS_ANY_DICT_r22 }, + { 3, 3, _GUARD_TOS_ANY_DICT_r33 }, + }, + }, + [_GUARD_TOS_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_DICT_r01 }, + { 1, 1, _GUARD_TOS_DICT_r11 }, + { 2, 2, _GUARD_TOS_DICT_r22 }, + { 3, 3, _GUARD_TOS_DICT_r33 }, + }, + }, + [_GUARD_TOS_FROZENDICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_FROZENDICT_r01 }, + { 1, 1, _GUARD_TOS_FROZENDICT_r11 }, + { 2, 2, _GUARD_TOS_FROZENDICT_r22 }, + { 3, 3, _GUARD_TOS_FROZENDICT_r33 }, + }, + }, + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_DICT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_DICT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_CHECK_FUNC] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_CHECK_FUNC_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_INIT_CALL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _BINARY_OP_SUBSCR_INIT_CALL_r01 }, + { 1, 1, _BINARY_OP_SUBSCR_INIT_CALL_r11 }, + { 1, 2, _BINARY_OP_SUBSCR_INIT_CALL_r21 }, + { 1, 3, _BINARY_OP_SUBSCR_INIT_CALL_r31 }, + }, + }, + [_LIST_APPEND] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _LIST_APPEND_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SET_ADD] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _SET_ADD_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_SUBSCR] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 3, _STORE_SUBSCR_r30 }, + }, + }, + [_STORE_SUBSCR_LIST_INT] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 3, _STORE_SUBSCR_LIST_INT_r32 }, + }, + }, + [_STORE_SUBSCR_DICT] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _STORE_SUBSCR_DICT_r31 }, + }, + }, + [_STORE_SUBSCR_DICT_KNOWN_HASH] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _STORE_SUBSCR_DICT_KNOWN_HASH_r31 }, + }, + }, + [_DELETE_SUBSCR] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 2, _DELETE_SUBSCR_r20 }, + { -1, -1, -1 }, + }, + }, + [_CALL_INTRINSIC_1] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _CALL_INTRINSIC_1_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_INTRINSIC_2] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _CALL_INTRINSIC_2_r23 }, + { -1, -1, -1 }, + }, + }, + [_MAKE_HEAP_SAFE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _MAKE_HEAP_SAFE_r01 }, + { 1, 1, _MAKE_HEAP_SAFE_r11 }, + { 2, 2, _MAKE_HEAP_SAFE_r22 }, + { 3, 3, _MAKE_HEAP_SAFE_r33 }, + }, + }, + [_RETURN_VALUE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _RETURN_VALUE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GET_AITER] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _GET_AITER_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GET_ANEXT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _GET_ANEXT_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GET_AWAITABLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _GET_AWAITABLE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SEND_GEN_FRAME] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _SEND_GEN_FRAME_r33 }, + }, + }, + [_YIELD_VALUE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _YIELD_VALUE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_POP_EXCEPT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _POP_EXCEPT_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_COMMON_CONSTANT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_COMMON_CONSTANT_r01 }, + { 2, 1, _LOAD_COMMON_CONSTANT_r12 }, + { 3, 2, _LOAD_COMMON_CONSTANT_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_BUILD_CLASS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_BUILD_CLASS_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_NAME] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _STORE_NAME_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DELETE_NAME] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _DELETE_NAME_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _UNPACK_SEQUENCE_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_TWO_TUPLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _UNPACK_SEQUENCE_TWO_TUPLE_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02 }, + { 2, 1, _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12 }, + { 3, 2, _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE] = { + .best = { 0, 1, 1, 1 }, + .entries = { + { 3, 0, _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03 }, + { 3, 1, _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_TUPLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _UNPACK_SEQUENCE_TUPLE_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_UNIQUE_TUPLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_LIST] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _UNPACK_SEQUENCE_LIST_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_EX] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _UNPACK_EX_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_ATTR] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 2, _STORE_ATTR_r20 }, + { -1, -1, -1 }, + }, + }, + [_DELETE_ATTR] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _DELETE_ATTR_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_GLOBAL] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _STORE_GLOBAL_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DELETE_GLOBAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _DELETE_GLOBAL_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_LOCALS] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_LOCALS_r01 }, + { 2, 1, _LOAD_LOCALS_r12 }, + { 3, 2, _LOAD_LOCALS_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_NAME] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_NAME_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_GLOBAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _LOAD_GLOBAL_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_PUSH_NULL_CONDITIONAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _PUSH_NULL_CONDITIONAL_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_GLOBALS_VERSION] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_GLOBALS_VERSION_r00 }, + { 1, 1, _GUARD_GLOBALS_VERSION_r11 }, + { 2, 2, _GUARD_GLOBALS_VERSION_r22 }, + { 3, 3, _GUARD_GLOBALS_VERSION_r33 }, + }, + }, + [_LOAD_GLOBAL_MODULE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_GLOBAL_MODULE_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_GLOBAL_BUILTINS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_GLOBAL_BUILTINS_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DELETE_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _DELETE_FAST_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAKE_CELL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _MAKE_CELL_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DELETE_DEREF] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _DELETE_DEREF_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FROM_DICT_OR_DEREF] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_FROM_DICT_OR_DEREF_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_DEREF] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_DEREF_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_DEREF] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _STORE_DEREF_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_COPY_FREE_VARS] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _COPY_FREE_VARS_r00 }, + { 1, 1, _COPY_FREE_VARS_r11 }, + { 2, 2, _COPY_FREE_VARS_r22 }, + { 3, 3, _COPY_FREE_VARS_r33 }, + }, + }, + [_BUILD_STRING] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_STRING_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_INTERPOLATION] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_INTERPOLATION_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_TEMPLATE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _BUILD_TEMPLATE_r21 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_TUPLE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_TUPLE_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_LIST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_LIST_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LIST_EXTEND] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LIST_EXTEND_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SET_UPDATE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _SET_UPDATE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_SET] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_SET_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_MAP] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_MAP_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SETUP_ANNOTATIONS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _SETUP_ANNOTATIONS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DICT_UPDATE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _DICT_UPDATE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DICT_MERGE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _DICT_MERGE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAP_ADD] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 2, _MAP_ADD_r20 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SUPER_ATTR_ATTR] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _LOAD_SUPER_ATTR_ATTR_r31 }, + }, + }, + [_GUARD_NOS_TYPE_VERSION] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_TYPE_VERSION_r02 }, + { 2, 1, _GUARD_NOS_TYPE_VERSION_r12 }, + { 2, 2, _GUARD_NOS_TYPE_VERSION_r22 }, + { 3, 3, _GUARD_NOS_TYPE_VERSION_r33 }, + }, + }, + [_GUARD_LOAD_SUPER_ATTR_METHOD] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_LOAD_SUPER_ATTR_METHOD_r03 }, + { 3, 1, _GUARD_LOAD_SUPER_ATTR_METHOD_r13 }, + { 3, 2, _GUARD_LOAD_SUPER_ATTR_METHOD_r23 }, + { 3, 3, _GUARD_LOAD_SUPER_ATTR_METHOD_r33 }, + }, + }, + [_LOAD_SUPER_ATTR_METHOD] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 3, _LOAD_SUPER_ATTR_METHOD_r32 }, + }, + }, + [_LOAD_ATTR] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _LOAD_ATTR_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_TYPE_VERSION] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TYPE_VERSION_r01 }, + { 1, 1, _GUARD_TYPE_VERSION_r11 }, + { 2, 2, _GUARD_TYPE_VERSION_r22 }, + { 3, 3, _GUARD_TYPE_VERSION_r33 }, + }, + }, + [_GUARD_TYPE_VERSION_LOCKED] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TYPE_VERSION_LOCKED_r01 }, + { 1, 1, _GUARD_TYPE_VERSION_LOCKED_r11 }, + { 2, 2, _GUARD_TYPE_VERSION_LOCKED_r22 }, + { 3, 3, _GUARD_TYPE_VERSION_LOCKED_r33 }, + }, + }, + [_GUARD_TYPE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TYPE_r01 }, + { 1, 1, _GUARD_TYPE_r11 }, + { 2, 2, _GUARD_TYPE_r22 }, + { 3, 3, _GUARD_TYPE_r33 }, + }, + }, + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _CHECK_MANAGED_OBJECT_HAS_VALUES_r01 }, + { 1, 1, _CHECK_MANAGED_OBJECT_HAS_VALUES_r11 }, + { 2, 2, _CHECK_MANAGED_OBJECT_HAS_VALUES_r22 }, + { 3, 3, _CHECK_MANAGED_OBJECT_HAS_VALUES_r33 }, + }, + }, + [_LOAD_ATTR_INSTANCE_VALUE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _LOAD_ATTR_INSTANCE_VALUE_r02 }, + { 2, 1, _LOAD_ATTR_INSTANCE_VALUE_r12 }, + { 3, 2, _LOAD_ATTR_INSTANCE_VALUE_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_MODULE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _LOAD_ATTR_MODULE_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_WITH_HINT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _LOAD_ATTR_WITH_HINT_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_SLOT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _LOAD_ATTR_SLOT_r02 }, + { 2, 1, _LOAD_ATTR_SLOT_r12 }, + { 3, 2, _LOAD_ATTR_SLOT_r23 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_ATTR_CLASS] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _CHECK_ATTR_CLASS_r01 }, + { 1, 1, _CHECK_ATTR_CLASS_r11 }, + { 2, 2, _CHECK_ATTR_CLASS_r22 }, + { 3, 3, _CHECK_ATTR_CLASS_r33 }, + }, + }, + [_LOAD_ATTR_CLASS] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_CLASS_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_PROPERTY_FRAME] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _LOAD_ATTR_PROPERTY_FRAME_r01 }, + { 1, 1, _LOAD_ATTR_PROPERTY_FRAME_r11 }, + { 2, 2, _LOAD_ATTR_PROPERTY_FRAME_r22 }, + { 3, 3, _LOAD_ATTR_PROPERTY_FRAME_r33 }, + }, + }, + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_DORV_NO_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_DORV_NO_DICT_r01 }, + { 1, 1, _GUARD_DORV_NO_DICT_r11 }, + { 2, 2, _GUARD_DORV_NO_DICT_r22 }, + { 3, 3, _GUARD_DORV_NO_DICT_r33 }, + }, + }, + [_STORE_ATTR_INSTANCE_VALUE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _STORE_ATTR_INSTANCE_VALUE_r21 }, + { -1, -1, -1 }, + }, + }, + [_LOCK_OBJECT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _LOCK_OBJECT_r01 }, + { 1, 1, _LOCK_OBJECT_r11 }, + { 2, 2, _LOCK_OBJECT_r22 }, + { 3, 3, _LOCK_OBJECT_r33 }, + }, + }, + [_STORE_ATTR_WITH_HINT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _STORE_ATTR_WITH_HINT_r21 }, + { -1, -1, -1 }, + }, + }, + [_STORE_ATTR_SLOT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _STORE_ATTR_SLOT_r21 }, + { -1, -1, -1 }, + }, + }, + [_COMPARE_OP] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _COMPARE_OP_r21 }, + { -1, -1, -1 }, + }, + }, + [_COMPARE_OP_FLOAT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _COMPARE_OP_FLOAT_r03 }, + { 3, 1, _COMPARE_OP_FLOAT_r13 }, + { 3, 2, _COMPARE_OP_FLOAT_r23 }, + { -1, -1, -1 }, + }, + }, + [_COMPARE_OP_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _COMPARE_OP_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_COMPARE_OP_STR] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _COMPARE_OP_STR_r23 }, + { -1, -1, -1 }, + }, + }, + [_IS_OP] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _IS_OP_r03 }, + { 3, 1, _IS_OP_r13 }, + { 3, 2, _IS_OP_r23 }, + { -1, -1, -1 }, + }, + }, + [_CONTAINS_OP] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _CONTAINS_OP_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_TOS_ANY_SET] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_ANY_SET_r01 }, + { 1, 1, _GUARD_TOS_ANY_SET_r11 }, + { 2, 2, _GUARD_TOS_ANY_SET_r22 }, + { 3, 3, _GUARD_TOS_ANY_SET_r33 }, + }, + }, + [_GUARD_TOS_SET] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_SET_r01 }, + { 1, 1, _GUARD_TOS_SET_r11 }, + { 2, 2, _GUARD_TOS_SET_r22 }, + { 3, 3, _GUARD_TOS_SET_r33 }, + }, + }, + [_GUARD_TOS_FROZENSET] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_FROZENSET_r01 }, + { 1, 1, _GUARD_TOS_FROZENSET_r11 }, + { 2, 2, _GUARD_TOS_FROZENSET_r22 }, + { 3, 3, _GUARD_TOS_FROZENSET_r33 }, + }, + }, + [_CONTAINS_OP_SET] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _CONTAINS_OP_SET_r23 }, + { -1, -1, -1 }, + }, + }, + [_CONTAINS_OP_DICT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _CONTAINS_OP_DICT_r23 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_EG_MATCH] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 2, _CHECK_EG_MATCH_r22 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_EXC_MATCH] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 2, _CHECK_EXC_MATCH_r22 }, + { -1, -1, -1 }, + }, + }, + [_IMPORT_NAME] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _IMPORT_NAME_r21 }, + { -1, -1, -1 }, + }, + }, + [_IMPORT_FROM] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _IMPORT_FROM_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_IS_NONE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _IS_NONE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GET_LEN] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _GET_LEN_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MATCH_CLASS] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _MATCH_CLASS_r33 }, + }, + }, + [_MATCH_MAPPING] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _MATCH_MAPPING_r02 }, + { 2, 1, _MATCH_MAPPING_r12 }, + { 3, 2, _MATCH_MAPPING_r23 }, + { -1, -1, -1 }, + }, + }, + [_MATCH_SEQUENCE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _MATCH_SEQUENCE_r02 }, + { 2, 1, _MATCH_SEQUENCE_r12 }, + { 3, 2, _MATCH_SEQUENCE_r23 }, + { -1, -1, -1 }, + }, + }, + [_MATCH_KEYS] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _MATCH_KEYS_r23 }, + { -1, -1, -1 }, + }, + }, + [_GET_ITER] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _GET_ITER_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_ITERATOR] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_ITERATOR_r01 }, + { 1, 1, _GUARD_ITERATOR_r11 }, + { 2, 2, _GUARD_ITERATOR_r22 }, + { 3, 3, _GUARD_ITERATOR_r33 }, + }, + }, + [_GUARD_ITER_VIRTUAL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_ITER_VIRTUAL_r01 }, + { 1, 1, _GUARD_ITER_VIRTUAL_r11 }, + { 2, 2, _GUARD_ITER_VIRTUAL_r22 }, + { 3, 3, _GUARD_ITER_VIRTUAL_r33 }, + }, + }, + [_PUSH_TAGGED_ZERO] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _PUSH_TAGGED_ZERO_r01 }, + { 2, 1, _PUSH_TAGGED_ZERO_r12 }, + { 3, 2, _PUSH_TAGGED_ZERO_r23 }, + { -1, -1, -1 }, + }, + }, + [_GET_ITER_TRAD] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _GET_ITER_TRAD_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_FOR_ITER_TIER_TWO] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _FOR_ITER_TIER_TWO_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_ITER_VIRTUAL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_ITER_VIRTUAL_r02 }, + { 2, 1, _GUARD_NOS_ITER_VIRTUAL_r12 }, + { 2, 2, _GUARD_NOS_ITER_VIRTUAL_r22 }, + { 3, 3, _GUARD_NOS_ITER_VIRTUAL_r33 }, + }, + }, + [_FOR_ITER_VIRTUAL_TIER_TWO] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _FOR_ITER_VIRTUAL_TIER_TWO_r23 }, + { -1, -1, -1 }, + }, + }, + [_ITER_CHECK_LIST] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _ITER_CHECK_LIST_r02 }, + { 2, 1, _ITER_CHECK_LIST_r12 }, + { 2, 2, _ITER_CHECK_LIST_r22 }, + { 3, 3, _ITER_CHECK_LIST_r33 }, + }, + }, + [_GUARD_NOT_EXHAUSTED_LIST] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOT_EXHAUSTED_LIST_r02 }, + { 2, 1, _GUARD_NOT_EXHAUSTED_LIST_r12 }, + { 2, 2, _GUARD_NOT_EXHAUSTED_LIST_r22 }, + { 3, 3, _GUARD_NOT_EXHAUSTED_LIST_r33 }, + }, + }, + [_ITER_NEXT_LIST_TIER_TWO] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _ITER_NEXT_LIST_TIER_TWO_r23 }, + { -1, -1, -1 }, + }, + }, + [_ITER_CHECK_TUPLE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _ITER_CHECK_TUPLE_r02 }, + { 2, 1, _ITER_CHECK_TUPLE_r12 }, + { 2, 2, _ITER_CHECK_TUPLE_r22 }, + { 3, 3, _ITER_CHECK_TUPLE_r33 }, + }, + }, + [_GUARD_NOT_EXHAUSTED_TUPLE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOT_EXHAUSTED_TUPLE_r02 }, + { 2, 1, _GUARD_NOT_EXHAUSTED_TUPLE_r12 }, + { 2, 2, _GUARD_NOT_EXHAUSTED_TUPLE_r22 }, + { 3, 3, _GUARD_NOT_EXHAUSTED_TUPLE_r33 }, + }, + }, + [_ITER_NEXT_TUPLE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _ITER_NEXT_TUPLE_r03 }, + { 3, 1, _ITER_NEXT_TUPLE_r13 }, + { 3, 2, _ITER_NEXT_TUPLE_r23 }, + { -1, -1, -1 }, + }, + }, + [_ITER_CHECK_RANGE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _ITER_CHECK_RANGE_r02 }, + { 2, 1, _ITER_CHECK_RANGE_r12 }, + { 2, 2, _ITER_CHECK_RANGE_r22 }, + { 3, 3, _ITER_CHECK_RANGE_r33 }, + }, + }, + [_GUARD_NOT_EXHAUSTED_RANGE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOT_EXHAUSTED_RANGE_r02 }, + { 2, 1, _GUARD_NOT_EXHAUSTED_RANGE_r12 }, + { 2, 2, _GUARD_NOT_EXHAUSTED_RANGE_r22 }, + { 3, 3, _GUARD_NOT_EXHAUSTED_RANGE_r33 }, + }, + }, + [_ITER_NEXT_RANGE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _ITER_NEXT_RANGE_r03 }, + { 3, 1, _ITER_NEXT_RANGE_r13 }, + { 3, 2, _ITER_NEXT_RANGE_r23 }, + { -1, -1, -1 }, + }, + }, + [_FOR_ITER_GEN_FRAME] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _FOR_ITER_GEN_FRAME_r03 }, + { 3, 1, _FOR_ITER_GEN_FRAME_r13 }, + { 3, 2, _FOR_ITER_GEN_FRAME_r23 }, + { -1, -1, -1 }, + }, + }, + [_INSERT_NULL] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _INSERT_NULL_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SPECIAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _LOAD_SPECIAL_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_WITH_EXCEPT_START] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _WITH_EXCEPT_START_r33 }, + }, + }, + [_PUSH_EXC_INFO] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _PUSH_EXC_INFO_r02 }, + { 2, 1, _PUSH_EXC_INFO_r12 }, + { 3, 2, _PUSH_EXC_INFO_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01 }, + { 1, 1, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 }, + { 2, 2, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 }, + { 3, 3, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 }, + }, + }, + [_GUARD_KEYS_VERSION] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_KEYS_VERSION_r01 }, + { 1, 1, _GUARD_KEYS_VERSION_r11 }, + { 2, 2, _GUARD_KEYS_VERSION_r22 }, + { 3, 3, _GUARD_KEYS_VERSION_r33 }, + }, + }, + [_LOAD_ATTR_METHOD_WITH_VALUES] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _LOAD_ATTR_METHOD_WITH_VALUES_r02 }, + { 2, 1, _LOAD_ATTR_METHOD_WITH_VALUES_r12 }, + { 3, 2, _LOAD_ATTR_METHOD_WITH_VALUES_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_METHOD_NO_DICT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _LOAD_ATTR_METHOD_NO_DICT_r02 }, + { 2, 1, _LOAD_ATTR_METHOD_NO_DICT_r12 }, + { 3, 2, _LOAD_ATTR_METHOD_NO_DICT_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_ATTR_METHOD_LAZY_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _CHECK_ATTR_METHOD_LAZY_DICT_r01 }, + { 1, 1, _CHECK_ATTR_METHOD_LAZY_DICT_r11 }, + { 2, 2, _CHECK_ATTR_METHOD_LAZY_DICT_r22 }, + { 3, 3, _CHECK_ATTR_METHOD_LAZY_DICT_r33 }, + }, + }, + [_LOAD_ATTR_METHOD_LAZY_DICT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _LOAD_ATTR_METHOD_LAZY_DICT_r02 }, + { 2, 1, _LOAD_ATTR_METHOD_LAZY_DICT_r12 }, + { 3, 2, _LOAD_ATTR_METHOD_LAZY_DICT_r23 }, + { -1, -1, -1 }, + }, + }, + [_MAYBE_EXPAND_METHOD] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _MAYBE_EXPAND_METHOD_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_PY_FRAME_GENERAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 1, _PY_FRAME_GENERAL_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_FUNCTION_VERSION] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_FUNCTION_VERSION_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_FUNCTION_VERSION_INLINE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_FUNCTION_VERSION_INLINE_r00 }, + { 1, 1, _CHECK_FUNCTION_VERSION_INLINE_r11 }, + { 2, 2, _CHECK_FUNCTION_VERSION_INLINE_r22 }, + { 3, 3, _CHECK_FUNCTION_VERSION_INLINE_r33 }, + }, + }, + [_CHECK_METHOD_VERSION] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_METHOD_VERSION_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_EXPAND_METHOD] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _EXPAND_METHOD_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_IS_NOT_PY_CALLABLE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_IS_NOT_PY_CALLABLE_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_NON_PY_GENERAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _CALL_NON_PY_GENERAL_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_PEP_523] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_PEP_523_r00 }, + { 1, 1, _CHECK_PEP_523_r11 }, + { 2, 2, _CHECK_PEP_523_r22 }, + { 3, 3, _CHECK_PEP_523_r33 }, + }, + }, + [_CHECK_FUNCTION_EXACT_ARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_FUNCTION_EXACT_ARGS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_STACK_SPACE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_STACK_SPACE_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_RECURSION_REMAINING] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_RECURSION_REMAINING_r00 }, + { 1, 1, _CHECK_RECURSION_REMAINING_r11 }, + { 2, 2, _CHECK_RECURSION_REMAINING_r22 }, + { 3, 3, _CHECK_RECURSION_REMAINING_r33 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_0] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_0_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_1] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_1_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_2] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_2_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_3] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_3_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_4] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_4_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_PUSH_FRAME] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _PUSH_FRAME_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_NULL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_NULL_r02 }, + { 2, 1, _GUARD_NOS_NULL_r12 }, + { 2, 2, _GUARD_NOS_NULL_r22 }, + { 3, 3, _GUARD_NOS_NULL_r33 }, + }, + }, + [_GUARD_NOS_NOT_NULL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_NOT_NULL_r02 }, + { 2, 1, _GUARD_NOS_NOT_NULL_r12 }, + { 2, 2, _GUARD_NOS_NOT_NULL_r22 }, + { 3, 3, _GUARD_NOS_NOT_NULL_r33 }, + }, + }, + [_GUARD_THIRD_NULL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_THIRD_NULL_r03 }, + { 3, 1, _GUARD_THIRD_NULL_r13 }, + { 3, 2, _GUARD_THIRD_NULL_r23 }, + { 3, 3, _GUARD_THIRD_NULL_r33 }, + }, + }, + [_GUARD_CALLABLE_TYPE_1] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_CALLABLE_TYPE_1_r03 }, + { 3, 1, _GUARD_CALLABLE_TYPE_1_r13 }, + { 3, 2, _GUARD_CALLABLE_TYPE_1_r23 }, + { 3, 3, _GUARD_CALLABLE_TYPE_1_r33 }, + }, + }, + [_CALL_TYPE_1] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _CALL_TYPE_1_r02 }, + { 2, 1, _CALL_TYPE_1_r12 }, + { 2, 2, _CALL_TYPE_1_r22 }, + { 2, 3, _CALL_TYPE_1_r32 }, + }, + }, + [_GUARD_CALLABLE_STR_1] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_CALLABLE_STR_1_r03 }, + { 3, 1, _GUARD_CALLABLE_STR_1_r13 }, + { 3, 2, _GUARD_CALLABLE_STR_1_r23 }, + { 3, 3, _GUARD_CALLABLE_STR_1_r33 }, + }, + }, + [_CALL_STR_1] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 3, _CALL_STR_1_r32 }, + }, + }, + [_GUARD_CALLABLE_TUPLE_1] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_CALLABLE_TUPLE_1_r03 }, + { 3, 1, _GUARD_CALLABLE_TUPLE_1_r13 }, + { 3, 2, _GUARD_CALLABLE_TUPLE_1_r23 }, + { 3, 3, _GUARD_CALLABLE_TUPLE_1_r33 }, + }, + }, + [_CALL_TUPLE_1] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 3, _CALL_TUPLE_1_r32 }, + }, + }, + [_CHECK_OBJECT] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_OBJECT_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_ALLOCATE_OBJECT] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _ALLOCATE_OBJECT_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CREATE_INIT_FRAME] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 1, _CREATE_INIT_FRAME_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_EXIT_INIT_CHECK] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _EXIT_INIT_CHECK_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_BUILTIN_CLASS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_BUILTIN_CLASS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_BUILTIN_CLASS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_BUILTIN_CLASS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_BUILTIN_O] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_BUILTIN_O_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_BUILTIN_O] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 3, 0, _CALL_BUILTIN_O_r03 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_BUILTIN_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_BUILTIN_FAST_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_BUILTIN_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_BUILTIN_FAST_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_BUILTIN_FAST_WITH_KEYWORDS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_LEN] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_CALLABLE_LEN_r03 }, + { 3, 1, _GUARD_CALLABLE_LEN_r13 }, + { 3, 2, _GUARD_CALLABLE_LEN_r23 }, + { 3, 3, _GUARD_CALLABLE_LEN_r33 }, + }, + }, + [_CALL_LEN] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _CALL_LEN_r33 }, + }, + }, + [_GUARD_CALLABLE_ISINSTANCE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_CALLABLE_ISINSTANCE_r03 }, + { 3, 1, _GUARD_CALLABLE_ISINSTANCE_r13 }, + { 3, 2, _GUARD_CALLABLE_ISINSTANCE_r23 }, + { 3, 3, _GUARD_CALLABLE_ISINSTANCE_r33 }, + }, + }, + [_CALL_ISINSTANCE] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _CALL_ISINSTANCE_r31 }, + }, + }, + [_GUARD_CALLABLE_LIST_APPEND] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_CALLABLE_LIST_APPEND_r03 }, + { 3, 1, _GUARD_CALLABLE_LIST_APPEND_r13 }, + { 3, 2, _GUARD_CALLABLE_LIST_APPEND_r23 }, + { 3, 3, _GUARD_CALLABLE_LIST_APPEND_r33 }, + }, + }, + [_CALL_LIST_APPEND] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _CALL_LIST_APPEND_r03 }, + { 3, 1, _CALL_LIST_APPEND_r13 }, + { 3, 2, _CALL_LIST_APPEND_r23 }, + { 3, 3, _CALL_LIST_APPEND_r33 }, + }, + }, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_O] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 3, 0, _CALL_METHOD_DESCRIPTOR_O_r03 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_RECURSION_LIMIT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_RECURSION_LIMIT_r00 }, + { 1, 1, _CHECK_RECURSION_LIMIT_r11 }, + { 2, 2, _CHECK_RECURSION_LIMIT_r22 }, + { 3, 3, _CHECK_RECURSION_LIMIT_r33 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_O_INLINE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 3, 0, _CALL_METHOD_DESCRIPTOR_O_INLINE_r03 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_NOARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 3, 0, _CALL_METHOD_DESCRIPTOR_NOARGS_r03 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 3, 0, _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAYBE_EXPAND_METHOD_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _MAYBE_EXPAND_METHOD_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_PY_FRAME_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _PY_FRAME_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_FUNCTION_VERSION_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _CHECK_FUNCTION_VERSION_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_METHOD_VERSION_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _CHECK_METHOD_VERSION_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_EXPAND_METHOD_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _EXPAND_METHOD_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_IS_NOT_PY_CALLABLE_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _CHECK_IS_NOT_PY_CALLABLE_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_KW_NON_PY] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _CALL_KW_NON_PY_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAKE_CALLARGS_A_TUPLE] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _MAKE_CALLARGS_A_TUPLE_r33 }, + }, + }, + [_CHECK_IS_PY_CALLABLE_EX] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _CHECK_IS_PY_CALLABLE_EX_r03 }, + { 3, 1, _CHECK_IS_PY_CALLABLE_EX_r13 }, + { 3, 2, _CHECK_IS_PY_CALLABLE_EX_r23 }, + { 3, 3, _CHECK_IS_PY_CALLABLE_EX_r33 }, + }, + }, + [_PY_FRAME_EX] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 1, _PY_FRAME_EX_r31 }, + }, + }, + [_CHECK_IS_NOT_PY_CALLABLE_EX] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _CHECK_IS_NOT_PY_CALLABLE_EX_r03 }, + { 3, 1, _CHECK_IS_NOT_PY_CALLABLE_EX_r13 }, + { 3, 2, _CHECK_IS_NOT_PY_CALLABLE_EX_r23 }, + { 3, 3, _CHECK_IS_NOT_PY_CALLABLE_EX_r33 }, + }, + }, + [_CALL_FUNCTION_EX_NON_PY_GENERAL] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _CALL_FUNCTION_EX_NON_PY_GENERAL_r31 }, + }, + }, + [_MAKE_FUNCTION] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _MAKE_FUNCTION_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SET_FUNCTION_ATTRIBUTE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SET_FUNCTION_ATTRIBUTE_r01 }, + { 1, 1, _SET_FUNCTION_ATTRIBUTE_r11 }, + { 1, 2, _SET_FUNCTION_ATTRIBUTE_r21 }, + { 2, 3, _SET_FUNCTION_ATTRIBUTE_r32 }, + }, + }, + [_RETURN_GENERATOR] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _RETURN_GENERATOR_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_SLICE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_SLICE_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CONVERT_VALUE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _CONVERT_VALUE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_FORMAT_SIMPLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _FORMAT_SIMPLE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_FORMAT_WITH_SPEC] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _FORMAT_WITH_SPEC_r21 }, + { -1, -1, -1 }, + }, + }, + [_COPY_1] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _COPY_1_r02 }, + { 2, 1, _COPY_1_r12 }, + { 3, 2, _COPY_1_r23 }, + { -1, -1, -1 }, + }, + }, + [_COPY_2] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _COPY_2_r03 }, + { 3, 1, _COPY_2_r13 }, + { 3, 2, _COPY_2_r23 }, + { -1, -1, -1 }, + }, + }, + [_COPY_3] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _COPY_3_r03 }, + { 3, 1, _COPY_3_r13 }, + { 3, 2, _COPY_3_r23 }, + { 3, 3, _COPY_3_r33 }, + }, + }, + [_COPY] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _COPY_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_r23 }, + { -1, -1, -1 }, + }, + }, + [_SWAP_2] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _SWAP_2_r02 }, + { 2, 1, _SWAP_2_r12 }, + { 2, 2, _SWAP_2_r22 }, + { 3, 3, _SWAP_2_r33 }, + }, + }, + [_SWAP_3] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _SWAP_3_r03 }, + { 3, 1, _SWAP_3_r13 }, + { 3, 2, _SWAP_3_r23 }, + { 3, 3, _SWAP_3_r33 }, + }, + }, + [_SWAP] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _SWAP_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_IS_TRUE_POP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IS_TRUE_POP_r00 }, + { 0, 0, _GUARD_IS_TRUE_POP_r10 }, + { 1, 1, _GUARD_IS_TRUE_POP_r21 }, + { 2, 2, _GUARD_IS_TRUE_POP_r32 }, + }, + }, + [_GUARD_IS_FALSE_POP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IS_FALSE_POP_r00 }, + { 0, 0, _GUARD_IS_FALSE_POP_r10 }, + { 1, 1, _GUARD_IS_FALSE_POP_r21 }, + { 2, 2, _GUARD_IS_FALSE_POP_r32 }, + }, + }, + [_GUARD_BIT_IS_SET_POP_4] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_SET_POP_4_r00 }, + { 0, 0, _GUARD_BIT_IS_SET_POP_4_r10 }, + { 1, 1, _GUARD_BIT_IS_SET_POP_4_r21 }, + { 2, 2, _GUARD_BIT_IS_SET_POP_4_r32 }, + }, + }, + [_GUARD_BIT_IS_SET_POP_5] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_SET_POP_5_r00 }, + { 0, 0, _GUARD_BIT_IS_SET_POP_5_r10 }, + { 1, 1, _GUARD_BIT_IS_SET_POP_5_r21 }, + { 2, 2, _GUARD_BIT_IS_SET_POP_5_r32 }, + }, + }, + [_GUARD_BIT_IS_SET_POP_6] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_SET_POP_6_r00 }, + { 0, 0, _GUARD_BIT_IS_SET_POP_6_r10 }, + { 1, 1, _GUARD_BIT_IS_SET_POP_6_r21 }, + { 2, 2, _GUARD_BIT_IS_SET_POP_6_r32 }, + }, + }, + [_GUARD_BIT_IS_SET_POP_7] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_SET_POP_7_r00 }, + { 0, 0, _GUARD_BIT_IS_SET_POP_7_r10 }, + { 1, 1, _GUARD_BIT_IS_SET_POP_7_r21 }, + { 2, 2, _GUARD_BIT_IS_SET_POP_7_r32 }, + }, + }, + [_GUARD_BIT_IS_SET_POP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_SET_POP_r00 }, + { 0, 0, _GUARD_BIT_IS_SET_POP_r10 }, + { 1, 1, _GUARD_BIT_IS_SET_POP_r21 }, + { 2, 2, _GUARD_BIT_IS_SET_POP_r32 }, + }, + }, + [_GUARD_BIT_IS_UNSET_POP_4] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_UNSET_POP_4_r00 }, + { 0, 0, _GUARD_BIT_IS_UNSET_POP_4_r10 }, + { 1, 1, _GUARD_BIT_IS_UNSET_POP_4_r21 }, + { 2, 2, _GUARD_BIT_IS_UNSET_POP_4_r32 }, + }, + }, + [_GUARD_BIT_IS_UNSET_POP_5] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_UNSET_POP_5_r00 }, + { 0, 0, _GUARD_BIT_IS_UNSET_POP_5_r10 }, + { 1, 1, _GUARD_BIT_IS_UNSET_POP_5_r21 }, + { 2, 2, _GUARD_BIT_IS_UNSET_POP_5_r32 }, + }, + }, + [_GUARD_BIT_IS_UNSET_POP_6] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_UNSET_POP_6_r00 }, + { 0, 0, _GUARD_BIT_IS_UNSET_POP_6_r10 }, + { 1, 1, _GUARD_BIT_IS_UNSET_POP_6_r21 }, + { 2, 2, _GUARD_BIT_IS_UNSET_POP_6_r32 }, + }, + }, + [_GUARD_BIT_IS_UNSET_POP_7] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_UNSET_POP_7_r00 }, + { 0, 0, _GUARD_BIT_IS_UNSET_POP_7_r10 }, + { 1, 1, _GUARD_BIT_IS_UNSET_POP_7_r21 }, + { 2, 2, _GUARD_BIT_IS_UNSET_POP_7_r32 }, + }, + }, + [_GUARD_BIT_IS_UNSET_POP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_UNSET_POP_r00 }, + { 0, 0, _GUARD_BIT_IS_UNSET_POP_r10 }, + { 1, 1, _GUARD_BIT_IS_UNSET_POP_r21 }, + { 2, 2, _GUARD_BIT_IS_UNSET_POP_r32 }, + }, + }, + [_GUARD_IS_NONE_POP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IS_NONE_POP_r00 }, + { 0, 0, _GUARD_IS_NONE_POP_r10 }, + { 1, 1, _GUARD_IS_NONE_POP_r21 }, + { 2, 2, _GUARD_IS_NONE_POP_r32 }, + }, + }, + [_GUARD_IS_NOT_NONE_POP] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _GUARD_IS_NOT_NONE_POP_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_JUMP_TO_TOP] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _JUMP_TO_TOP_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SET_IP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _SET_IP_r00 }, + { 1, 1, _SET_IP_r11 }, + { 2, 2, _SET_IP_r22 }, + { 3, 3, _SET_IP_r33 }, + }, + }, + [_CHECK_STACK_SPACE_OPERAND] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_STACK_SPACE_OPERAND_r00 }, + { 1, 1, _CHECK_STACK_SPACE_OPERAND_r11 }, + { 2, 2, _CHECK_STACK_SPACE_OPERAND_r22 }, + { 3, 3, _CHECK_STACK_SPACE_OPERAND_r33 }, + }, + }, + [_SAVE_RETURN_OFFSET] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _SAVE_RETURN_OFFSET_r00 }, + { 1, 1, _SAVE_RETURN_OFFSET_r11 }, + { 2, 2, _SAVE_RETURN_OFFSET_r22 }, + { 3, 3, _SAVE_RETURN_OFFSET_r33 }, + }, + }, + [_EXIT_TRACE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _EXIT_TRACE_r00 }, + { 0, 0, _EXIT_TRACE_r10 }, + { 0, 0, _EXIT_TRACE_r20 }, + { 0, 0, _EXIT_TRACE_r30 }, + }, + }, + [_DYNAMIC_EXIT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _DYNAMIC_EXIT_r00 }, + { 0, 0, _DYNAMIC_EXIT_r10 }, + { 0, 0, _DYNAMIC_EXIT_r20 }, + { 0, 0, _DYNAMIC_EXIT_r30 }, + }, + }, + [_CHECK_VALIDITY] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_VALIDITY_r00 }, + { 1, 1, _CHECK_VALIDITY_r11 }, + { 2, 2, _CHECK_VALIDITY_r22 }, + { 3, 3, _CHECK_VALIDITY_r33 }, + }, + }, + [_LOAD_CONST_INLINE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_CONST_INLINE_r01 }, + { 2, 1, _LOAD_CONST_INLINE_r12 }, + { 3, 2, _LOAD_CONST_INLINE_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_CONST_INLINE_BORROW] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_CONST_INLINE_BORROW_r01 }, + { 2, 1, _LOAD_CONST_INLINE_BORROW_r12 }, + { 3, 2, _LOAD_CONST_INLINE_BORROW_r23 }, + { -1, -1, -1 }, + }, + }, + [_RROT_3] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _RROT_3_r03 }, + { 3, 1, _RROT_3_r13 }, + { 3, 2, _RROT_3_r23 }, + { 3, 3, _RROT_3_r33 }, + }, + }, + [_START_EXECUTOR] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _START_EXECUTOR_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAKE_WARM] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _MAKE_WARM_r00 }, + { 1, 1, _MAKE_WARM_r11 }, + { 2, 2, _MAKE_WARM_r22 }, + { 3, 3, _MAKE_WARM_r33 }, + }, + }, + [_FATAL_ERROR] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _FATAL_ERROR_r00 }, + { 1, 1, _FATAL_ERROR_r11 }, + { 2, 2, _FATAL_ERROR_r22 }, + { 3, 3, _FATAL_ERROR_r33 }, + }, + }, + [_DEOPT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _DEOPT_r00 }, + { 0, 0, _DEOPT_r10 }, + { 0, 0, _DEOPT_r20 }, + { 0, 0, _DEOPT_r30 }, + }, + }, + [_HANDLE_PENDING_AND_DEOPT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _HANDLE_PENDING_AND_DEOPT_r00 }, + { 0, 0, _HANDLE_PENDING_AND_DEOPT_r10 }, + { 0, 0, _HANDLE_PENDING_AND_DEOPT_r20 }, + { 0, 0, _HANDLE_PENDING_AND_DEOPT_r30 }, + }, + }, + [_ERROR_POP_N] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _ERROR_POP_N_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_TIER2_RESUME_CHECK] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _TIER2_RESUME_CHECK_r00 }, + { 1, 1, _TIER2_RESUME_CHECK_r11 }, + { 2, 2, _TIER2_RESUME_CHECK_r22 }, + { 3, 3, _TIER2_RESUME_CHECK_r33 }, + }, + }, + [_COLD_EXIT] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _COLD_EXIT_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_COLD_DYNAMIC_EXIT] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _COLD_DYNAMIC_EXIT_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CODE_VERSION__PUSH_FRAME] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_CODE_VERSION__PUSH_FRAME_r00 }, + { 1, 1, _GUARD_CODE_VERSION__PUSH_FRAME_r11 }, + { 2, 2, _GUARD_CODE_VERSION__PUSH_FRAME_r22 }, + { 3, 3, _GUARD_CODE_VERSION__PUSH_FRAME_r33 }, + }, + }, + [_GUARD_CODE_VERSION_YIELD_VALUE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_CODE_VERSION_YIELD_VALUE_r00 }, + { 1, 1, _GUARD_CODE_VERSION_YIELD_VALUE_r11 }, + { 2, 2, _GUARD_CODE_VERSION_YIELD_VALUE_r22 }, + { 3, 3, _GUARD_CODE_VERSION_YIELD_VALUE_r33 }, + }, + }, + [_GUARD_CODE_VERSION_RETURN_VALUE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_CODE_VERSION_RETURN_VALUE_r00 }, + { 1, 1, _GUARD_CODE_VERSION_RETURN_VALUE_r11 }, + { 2, 2, _GUARD_CODE_VERSION_RETURN_VALUE_r22 }, + { 3, 3, _GUARD_CODE_VERSION_RETURN_VALUE_r33 }, + }, + }, + [_GUARD_CODE_VERSION_RETURN_GENERATOR] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_CODE_VERSION_RETURN_GENERATOR_r00 }, + { 1, 1, _GUARD_CODE_VERSION_RETURN_GENERATOR_r11 }, + { 2, 2, _GUARD_CODE_VERSION_RETURN_GENERATOR_r22 }, + { 3, 3, _GUARD_CODE_VERSION_RETURN_GENERATOR_r33 }, + }, + }, + [_GUARD_IP__PUSH_FRAME] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IP__PUSH_FRAME_r00 }, + { 1, 1, _GUARD_IP__PUSH_FRAME_r11 }, + { 2, 2, _GUARD_IP__PUSH_FRAME_r22 }, + { 3, 3, _GUARD_IP__PUSH_FRAME_r33 }, + }, + }, + [_GUARD_IP_YIELD_VALUE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IP_YIELD_VALUE_r00 }, + { 1, 1, _GUARD_IP_YIELD_VALUE_r11 }, + { 2, 2, _GUARD_IP_YIELD_VALUE_r22 }, + { 3, 3, _GUARD_IP_YIELD_VALUE_r33 }, + }, + }, + [_GUARD_IP_RETURN_VALUE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IP_RETURN_VALUE_r00 }, + { 1, 1, _GUARD_IP_RETURN_VALUE_r11 }, + { 2, 2, _GUARD_IP_RETURN_VALUE_r22 }, + { 3, 3, _GUARD_IP_RETURN_VALUE_r33 }, + }, + }, + [_GUARD_IP_RETURN_GENERATOR] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IP_RETURN_GENERATOR_r00 }, + { 1, 1, _GUARD_IP_RETURN_GENERATOR_r11 }, + { 2, 2, _GUARD_IP_RETURN_GENERATOR_r22 }, + { 3, 3, _GUARD_IP_RETURN_GENERATOR_r33 }, + }, + }, +}; + +const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { + [_NOP_r00] = _NOP, + [_NOP_r11] = _NOP, + [_NOP_r22] = _NOP, + [_NOP_r33] = _NOP, + [_CHECK_PERIODIC_r00] = _CHECK_PERIODIC, + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00] = _CHECK_PERIODIC_IF_NOT_YIELD_FROM, + [_RESUME_CHECK_r00] = _RESUME_CHECK, + [_RESUME_CHECK_r11] = _RESUME_CHECK, + [_RESUME_CHECK_r22] = _RESUME_CHECK, + [_RESUME_CHECK_r33] = _RESUME_CHECK, + [_LOAD_FAST_CHECK_r01] = _LOAD_FAST_CHECK, + [_LOAD_FAST_CHECK_r12] = _LOAD_FAST_CHECK, + [_LOAD_FAST_CHECK_r23] = _LOAD_FAST_CHECK, + [_LOAD_FAST_0_r01] = _LOAD_FAST_0, + [_LOAD_FAST_0_r12] = _LOAD_FAST_0, + [_LOAD_FAST_0_r23] = _LOAD_FAST_0, + [_LOAD_FAST_1_r01] = _LOAD_FAST_1, + [_LOAD_FAST_1_r12] = _LOAD_FAST_1, + [_LOAD_FAST_1_r23] = _LOAD_FAST_1, + [_LOAD_FAST_2_r01] = _LOAD_FAST_2, + [_LOAD_FAST_2_r12] = _LOAD_FAST_2, + [_LOAD_FAST_2_r23] = _LOAD_FAST_2, + [_LOAD_FAST_3_r01] = _LOAD_FAST_3, + [_LOAD_FAST_3_r12] = _LOAD_FAST_3, + [_LOAD_FAST_3_r23] = _LOAD_FAST_3, + [_LOAD_FAST_4_r01] = _LOAD_FAST_4, + [_LOAD_FAST_4_r12] = _LOAD_FAST_4, + [_LOAD_FAST_4_r23] = _LOAD_FAST_4, + [_LOAD_FAST_5_r01] = _LOAD_FAST_5, + [_LOAD_FAST_5_r12] = _LOAD_FAST_5, + [_LOAD_FAST_5_r23] = _LOAD_FAST_5, + [_LOAD_FAST_6_r01] = _LOAD_FAST_6, + [_LOAD_FAST_6_r12] = _LOAD_FAST_6, + [_LOAD_FAST_6_r23] = _LOAD_FAST_6, + [_LOAD_FAST_7_r01] = _LOAD_FAST_7, + [_LOAD_FAST_7_r12] = _LOAD_FAST_7, + [_LOAD_FAST_7_r23] = _LOAD_FAST_7, + [_LOAD_FAST_r01] = _LOAD_FAST, + [_LOAD_FAST_r12] = _LOAD_FAST, + [_LOAD_FAST_r23] = _LOAD_FAST, + [_LOAD_FAST_BORROW_0_r01] = _LOAD_FAST_BORROW_0, + [_LOAD_FAST_BORROW_0_r12] = _LOAD_FAST_BORROW_0, + [_LOAD_FAST_BORROW_0_r23] = _LOAD_FAST_BORROW_0, + [_LOAD_FAST_BORROW_1_r01] = _LOAD_FAST_BORROW_1, + [_LOAD_FAST_BORROW_1_r12] = _LOAD_FAST_BORROW_1, + [_LOAD_FAST_BORROW_1_r23] = _LOAD_FAST_BORROW_1, + [_LOAD_FAST_BORROW_2_r01] = _LOAD_FAST_BORROW_2, + [_LOAD_FAST_BORROW_2_r12] = _LOAD_FAST_BORROW_2, + [_LOAD_FAST_BORROW_2_r23] = _LOAD_FAST_BORROW_2, + [_LOAD_FAST_BORROW_3_r01] = _LOAD_FAST_BORROW_3, + [_LOAD_FAST_BORROW_3_r12] = _LOAD_FAST_BORROW_3, + [_LOAD_FAST_BORROW_3_r23] = _LOAD_FAST_BORROW_3, + [_LOAD_FAST_BORROW_4_r01] = _LOAD_FAST_BORROW_4, + [_LOAD_FAST_BORROW_4_r12] = _LOAD_FAST_BORROW_4, + [_LOAD_FAST_BORROW_4_r23] = _LOAD_FAST_BORROW_4, + [_LOAD_FAST_BORROW_5_r01] = _LOAD_FAST_BORROW_5, + [_LOAD_FAST_BORROW_5_r12] = _LOAD_FAST_BORROW_5, + [_LOAD_FAST_BORROW_5_r23] = _LOAD_FAST_BORROW_5, + [_LOAD_FAST_BORROW_6_r01] = _LOAD_FAST_BORROW_6, + [_LOAD_FAST_BORROW_6_r12] = _LOAD_FAST_BORROW_6, + [_LOAD_FAST_BORROW_6_r23] = _LOAD_FAST_BORROW_6, + [_LOAD_FAST_BORROW_7_r01] = _LOAD_FAST_BORROW_7, + [_LOAD_FAST_BORROW_7_r12] = _LOAD_FAST_BORROW_7, + [_LOAD_FAST_BORROW_7_r23] = _LOAD_FAST_BORROW_7, + [_LOAD_FAST_BORROW_r01] = _LOAD_FAST_BORROW, + [_LOAD_FAST_BORROW_r12] = _LOAD_FAST_BORROW, + [_LOAD_FAST_BORROW_r23] = _LOAD_FAST_BORROW, + [_LOAD_FAST_AND_CLEAR_r01] = _LOAD_FAST_AND_CLEAR, + [_LOAD_FAST_AND_CLEAR_r12] = _LOAD_FAST_AND_CLEAR, + [_LOAD_FAST_AND_CLEAR_r23] = _LOAD_FAST_AND_CLEAR, + [_LOAD_CONST_r01] = _LOAD_CONST, + [_LOAD_CONST_r12] = _LOAD_CONST, + [_LOAD_CONST_r23] = _LOAD_CONST, + [_LOAD_SMALL_INT_0_r01] = _LOAD_SMALL_INT_0, + [_LOAD_SMALL_INT_0_r12] = _LOAD_SMALL_INT_0, + [_LOAD_SMALL_INT_0_r23] = _LOAD_SMALL_INT_0, + [_LOAD_SMALL_INT_1_r01] = _LOAD_SMALL_INT_1, + [_LOAD_SMALL_INT_1_r12] = _LOAD_SMALL_INT_1, + [_LOAD_SMALL_INT_1_r23] = _LOAD_SMALL_INT_1, + [_LOAD_SMALL_INT_2_r01] = _LOAD_SMALL_INT_2, + [_LOAD_SMALL_INT_2_r12] = _LOAD_SMALL_INT_2, + [_LOAD_SMALL_INT_2_r23] = _LOAD_SMALL_INT_2, + [_LOAD_SMALL_INT_3_r01] = _LOAD_SMALL_INT_3, + [_LOAD_SMALL_INT_3_r12] = _LOAD_SMALL_INT_3, + [_LOAD_SMALL_INT_3_r23] = _LOAD_SMALL_INT_3, + [_LOAD_SMALL_INT_r01] = _LOAD_SMALL_INT, + [_LOAD_SMALL_INT_r12] = _LOAD_SMALL_INT, + [_LOAD_SMALL_INT_r23] = _LOAD_SMALL_INT, + [_SWAP_FAST_0_r01] = _SWAP_FAST_0, + [_SWAP_FAST_0_r11] = _SWAP_FAST_0, + [_SWAP_FAST_0_r22] = _SWAP_FAST_0, + [_SWAP_FAST_0_r33] = _SWAP_FAST_0, + [_SWAP_FAST_1_r01] = _SWAP_FAST_1, + [_SWAP_FAST_1_r11] = _SWAP_FAST_1, + [_SWAP_FAST_1_r22] = _SWAP_FAST_1, + [_SWAP_FAST_1_r33] = _SWAP_FAST_1, + [_SWAP_FAST_2_r01] = _SWAP_FAST_2, + [_SWAP_FAST_2_r11] = _SWAP_FAST_2, + [_SWAP_FAST_2_r22] = _SWAP_FAST_2, + [_SWAP_FAST_2_r33] = _SWAP_FAST_2, + [_SWAP_FAST_3_r01] = _SWAP_FAST_3, + [_SWAP_FAST_3_r11] = _SWAP_FAST_3, + [_SWAP_FAST_3_r22] = _SWAP_FAST_3, + [_SWAP_FAST_3_r33] = _SWAP_FAST_3, + [_SWAP_FAST_4_r01] = _SWAP_FAST_4, + [_SWAP_FAST_4_r11] = _SWAP_FAST_4, + [_SWAP_FAST_4_r22] = _SWAP_FAST_4, + [_SWAP_FAST_4_r33] = _SWAP_FAST_4, + [_SWAP_FAST_5_r01] = _SWAP_FAST_5, + [_SWAP_FAST_5_r11] = _SWAP_FAST_5, + [_SWAP_FAST_5_r22] = _SWAP_FAST_5, + [_SWAP_FAST_5_r33] = _SWAP_FAST_5, + [_SWAP_FAST_6_r01] = _SWAP_FAST_6, + [_SWAP_FAST_6_r11] = _SWAP_FAST_6, + [_SWAP_FAST_6_r22] = _SWAP_FAST_6, + [_SWAP_FAST_6_r33] = _SWAP_FAST_6, + [_SWAP_FAST_7_r01] = _SWAP_FAST_7, + [_SWAP_FAST_7_r11] = _SWAP_FAST_7, + [_SWAP_FAST_7_r22] = _SWAP_FAST_7, + [_SWAP_FAST_7_r33] = _SWAP_FAST_7, + [_SWAP_FAST_r01] = _SWAP_FAST, + [_SWAP_FAST_r11] = _SWAP_FAST, + [_SWAP_FAST_r22] = _SWAP_FAST, + [_SWAP_FAST_r33] = _SWAP_FAST, + [_POP_TOP_r10] = _POP_TOP, + [_POP_TOP_NOP_r00] = _POP_TOP_NOP, + [_POP_TOP_NOP_r10] = _POP_TOP_NOP, + [_POP_TOP_NOP_r21] = _POP_TOP_NOP, + [_POP_TOP_NOP_r32] = _POP_TOP_NOP, + [_POP_TOP_INT_r00] = _POP_TOP_INT, + [_POP_TOP_INT_r10] = _POP_TOP_INT, + [_POP_TOP_INT_r21] = _POP_TOP_INT, + [_POP_TOP_INT_r32] = _POP_TOP_INT, + [_POP_TOP_FLOAT_r00] = _POP_TOP_FLOAT, + [_POP_TOP_FLOAT_r10] = _POP_TOP_FLOAT, + [_POP_TOP_FLOAT_r21] = _POP_TOP_FLOAT, + [_POP_TOP_FLOAT_r32] = _POP_TOP_FLOAT, + [_POP_TOP_UNICODE_r00] = _POP_TOP_UNICODE, + [_POP_TOP_UNICODE_r10] = _POP_TOP_UNICODE, + [_POP_TOP_UNICODE_r21] = _POP_TOP_UNICODE, + [_POP_TOP_UNICODE_r32] = _POP_TOP_UNICODE, + [_POP_TOP_OPARG_r00] = _POP_TOP_OPARG, + [_PUSH_NULL_r01] = _PUSH_NULL, + [_PUSH_NULL_r12] = _PUSH_NULL, + [_PUSH_NULL_r23] = _PUSH_NULL, + [_END_FOR_r10] = _END_FOR, + [_POP_ITER_r20] = _POP_ITER, + [_END_SEND_r31] = _END_SEND, + [_UNARY_NEGATIVE_r12] = _UNARY_NEGATIVE, + [_UNARY_NEGATIVE_FLOAT_INPLACE_r02] = _UNARY_NEGATIVE_FLOAT_INPLACE, + [_UNARY_NEGATIVE_FLOAT_INPLACE_r12] = _UNARY_NEGATIVE_FLOAT_INPLACE, + [_UNARY_NEGATIVE_FLOAT_INPLACE_r23] = _UNARY_NEGATIVE_FLOAT_INPLACE, + [_UNARY_NOT_r01] = _UNARY_NOT, + [_UNARY_NOT_r11] = _UNARY_NOT, + [_UNARY_NOT_r22] = _UNARY_NOT, + [_UNARY_NOT_r33] = _UNARY_NOT, + [_TO_BOOL_r11] = _TO_BOOL, + [_TO_BOOL_BOOL_r01] = _TO_BOOL_BOOL, + [_TO_BOOL_BOOL_r11] = _TO_BOOL_BOOL, + [_TO_BOOL_BOOL_r22] = _TO_BOOL_BOOL, + [_TO_BOOL_BOOL_r33] = _TO_BOOL_BOOL, + [_TO_BOOL_INT_r02] = _TO_BOOL_INT, + [_TO_BOOL_INT_r12] = _TO_BOOL_INT, + [_TO_BOOL_INT_r23] = _TO_BOOL_INT, + [_GUARD_NOS_LIST_r02] = _GUARD_NOS_LIST, + [_GUARD_NOS_LIST_r12] = _GUARD_NOS_LIST, + [_GUARD_NOS_LIST_r22] = _GUARD_NOS_LIST, + [_GUARD_NOS_LIST_r33] = _GUARD_NOS_LIST, + [_GUARD_TOS_LIST_r01] = _GUARD_TOS_LIST, + [_GUARD_TOS_LIST_r11] = _GUARD_TOS_LIST, + [_GUARD_TOS_LIST_r22] = _GUARD_TOS_LIST, + [_GUARD_TOS_LIST_r33] = _GUARD_TOS_LIST, + [_GUARD_TOS_SLICE_r01] = _GUARD_TOS_SLICE, + [_GUARD_TOS_SLICE_r11] = _GUARD_TOS_SLICE, + [_GUARD_TOS_SLICE_r22] = _GUARD_TOS_SLICE, + [_GUARD_TOS_SLICE_r33] = _GUARD_TOS_SLICE, + [_TO_BOOL_LIST_r02] = _TO_BOOL_LIST, + [_TO_BOOL_LIST_r12] = _TO_BOOL_LIST, + [_TO_BOOL_LIST_r23] = _TO_BOOL_LIST, + [_TO_BOOL_NONE_r01] = _TO_BOOL_NONE, + [_TO_BOOL_NONE_r11] = _TO_BOOL_NONE, + [_TO_BOOL_NONE_r22] = _TO_BOOL_NONE, + [_TO_BOOL_NONE_r33] = _TO_BOOL_NONE, + [_GUARD_NOS_COMPACT_ASCII_r02] = _GUARD_NOS_COMPACT_ASCII, + [_GUARD_NOS_COMPACT_ASCII_r12] = _GUARD_NOS_COMPACT_ASCII, + [_GUARD_NOS_COMPACT_ASCII_r22] = _GUARD_NOS_COMPACT_ASCII, + [_GUARD_NOS_COMPACT_ASCII_r33] = _GUARD_NOS_COMPACT_ASCII, + [_GUARD_NOS_UNICODE_r02] = _GUARD_NOS_UNICODE, + [_GUARD_NOS_UNICODE_r12] = _GUARD_NOS_UNICODE, + [_GUARD_NOS_UNICODE_r22] = _GUARD_NOS_UNICODE, + [_GUARD_NOS_UNICODE_r33] = _GUARD_NOS_UNICODE, + [_GUARD_TOS_UNICODE_r01] = _GUARD_TOS_UNICODE, + [_GUARD_TOS_UNICODE_r11] = _GUARD_TOS_UNICODE, + [_GUARD_TOS_UNICODE_r22] = _GUARD_TOS_UNICODE, + [_GUARD_TOS_UNICODE_r33] = _GUARD_TOS_UNICODE, + [_TO_BOOL_STR_r02] = _TO_BOOL_STR, + [_TO_BOOL_STR_r12] = _TO_BOOL_STR, + [_TO_BOOL_STR_r23] = _TO_BOOL_STR, + [_REPLACE_WITH_TRUE_r02] = _REPLACE_WITH_TRUE, + [_REPLACE_WITH_TRUE_r12] = _REPLACE_WITH_TRUE, + [_REPLACE_WITH_TRUE_r23] = _REPLACE_WITH_TRUE, + [_UNARY_INVERT_r12] = _UNARY_INVERT, + [_GUARD_NOS_INT_r02] = _GUARD_NOS_INT, + [_GUARD_NOS_INT_r12] = _GUARD_NOS_INT, + [_GUARD_NOS_INT_r22] = _GUARD_NOS_INT, + [_GUARD_NOS_INT_r33] = _GUARD_NOS_INT, + [_GUARD_TOS_INT_r01] = _GUARD_TOS_INT, + [_GUARD_TOS_INT_r11] = _GUARD_TOS_INT, + [_GUARD_TOS_INT_r22] = _GUARD_TOS_INT, + [_GUARD_TOS_INT_r33] = _GUARD_TOS_INT, + [_GUARD_NOS_OVERFLOWED_r02] = _GUARD_NOS_OVERFLOWED, + [_GUARD_NOS_OVERFLOWED_r12] = _GUARD_NOS_OVERFLOWED, + [_GUARD_NOS_OVERFLOWED_r22] = _GUARD_NOS_OVERFLOWED, + [_GUARD_NOS_OVERFLOWED_r33] = _GUARD_NOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r01] = _GUARD_TOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r11] = _GUARD_TOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r22] = _GUARD_TOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r33] = _GUARD_TOS_OVERFLOWED, + [_BINARY_OP_MULTIPLY_INT_r03] = _BINARY_OP_MULTIPLY_INT, + [_BINARY_OP_MULTIPLY_INT_r13] = _BINARY_OP_MULTIPLY_INT, + [_BINARY_OP_MULTIPLY_INT_r23] = _BINARY_OP_MULTIPLY_INT, + [_BINARY_OP_ADD_INT_r03] = _BINARY_OP_ADD_INT, + [_BINARY_OP_ADD_INT_r13] = _BINARY_OP_ADD_INT, + [_BINARY_OP_ADD_INT_r23] = _BINARY_OP_ADD_INT, + [_BINARY_OP_SUBTRACT_INT_r03] = _BINARY_OP_SUBTRACT_INT, + [_BINARY_OP_SUBTRACT_INT_r13] = _BINARY_OP_SUBTRACT_INT, + [_BINARY_OP_SUBTRACT_INT_r23] = _BINARY_OP_SUBTRACT_INT, + [_BINARY_OP_ADD_INT_INPLACE_r03] = _BINARY_OP_ADD_INT_INPLACE, + [_BINARY_OP_ADD_INT_INPLACE_r13] = _BINARY_OP_ADD_INT_INPLACE, + [_BINARY_OP_ADD_INT_INPLACE_r23] = _BINARY_OP_ADD_INT_INPLACE, + [_BINARY_OP_SUBTRACT_INT_INPLACE_r03] = _BINARY_OP_SUBTRACT_INT_INPLACE, + [_BINARY_OP_SUBTRACT_INT_INPLACE_r13] = _BINARY_OP_SUBTRACT_INT_INPLACE, + [_BINARY_OP_SUBTRACT_INT_INPLACE_r23] = _BINARY_OP_SUBTRACT_INT_INPLACE, + [_BINARY_OP_MULTIPLY_INT_INPLACE_r03] = _BINARY_OP_MULTIPLY_INT_INPLACE, + [_BINARY_OP_MULTIPLY_INT_INPLACE_r13] = _BINARY_OP_MULTIPLY_INT_INPLACE, + [_BINARY_OP_MULTIPLY_INT_INPLACE_r23] = _BINARY_OP_MULTIPLY_INT_INPLACE, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r03] = _BINARY_OP_ADD_INT_INPLACE_RIGHT, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r13] = _BINARY_OP_ADD_INT_INPLACE_RIGHT, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r23] = _BINARY_OP_ADD_INT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03] = _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13] = _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23] = _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03] = _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13] = _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23] = _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, + [_GUARD_NOS_FLOAT_r02] = _GUARD_NOS_FLOAT, + [_GUARD_NOS_FLOAT_r12] = _GUARD_NOS_FLOAT, + [_GUARD_NOS_FLOAT_r22] = _GUARD_NOS_FLOAT, + [_GUARD_NOS_FLOAT_r33] = _GUARD_NOS_FLOAT, + [_GUARD_TOS_FLOAT_r01] = _GUARD_TOS_FLOAT, + [_GUARD_TOS_FLOAT_r11] = _GUARD_TOS_FLOAT, + [_GUARD_TOS_FLOAT_r22] = _GUARD_TOS_FLOAT, + [_GUARD_TOS_FLOAT_r33] = _GUARD_TOS_FLOAT, + [_BINARY_OP_MULTIPLY_FLOAT_r03] = _BINARY_OP_MULTIPLY_FLOAT, + [_BINARY_OP_MULTIPLY_FLOAT_r13] = _BINARY_OP_MULTIPLY_FLOAT, + [_BINARY_OP_MULTIPLY_FLOAT_r23] = _BINARY_OP_MULTIPLY_FLOAT, + [_BINARY_OP_ADD_FLOAT_r03] = _BINARY_OP_ADD_FLOAT, + [_BINARY_OP_ADD_FLOAT_r13] = _BINARY_OP_ADD_FLOAT, + [_BINARY_OP_ADD_FLOAT_r23] = _BINARY_OP_ADD_FLOAT, + [_BINARY_OP_SUBTRACT_FLOAT_r03] = _BINARY_OP_SUBTRACT_FLOAT, + [_BINARY_OP_SUBTRACT_FLOAT_r13] = _BINARY_OP_SUBTRACT_FLOAT, + [_BINARY_OP_SUBTRACT_FLOAT_r23] = _BINARY_OP_SUBTRACT_FLOAT, + [_BINARY_OP_ADD_FLOAT_INPLACE_r03] = _BINARY_OP_ADD_FLOAT_INPLACE, + [_BINARY_OP_ADD_FLOAT_INPLACE_r13] = _BINARY_OP_ADD_FLOAT_INPLACE, + [_BINARY_OP_ADD_FLOAT_INPLACE_r23] = _BINARY_OP_ADD_FLOAT_INPLACE, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03] = _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13] = _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23] = _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_TRUEDIV_FLOAT_r23] = _BINARY_OP_TRUEDIV_FLOAT, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_ADD_UNICODE_r03] = _BINARY_OP_ADD_UNICODE, + [_BINARY_OP_ADD_UNICODE_r13] = _BINARY_OP_ADD_UNICODE, + [_BINARY_OP_ADD_UNICODE_r23] = _BINARY_OP_ADD_UNICODE, + [_BINARY_OP_INPLACE_ADD_UNICODE_r21] = _BINARY_OP_INPLACE_ADD_UNICODE, + [_GUARD_BINARY_OP_EXTEND_LHS_r02] = _GUARD_BINARY_OP_EXTEND_LHS, + [_GUARD_BINARY_OP_EXTEND_LHS_r12] = _GUARD_BINARY_OP_EXTEND_LHS, + [_GUARD_BINARY_OP_EXTEND_LHS_r22] = _GUARD_BINARY_OP_EXTEND_LHS, + [_GUARD_BINARY_OP_EXTEND_LHS_r33] = _GUARD_BINARY_OP_EXTEND_LHS, + [_GUARD_BINARY_OP_EXTEND_RHS_r02] = _GUARD_BINARY_OP_EXTEND_RHS, + [_GUARD_BINARY_OP_EXTEND_RHS_r12] = _GUARD_BINARY_OP_EXTEND_RHS, + [_GUARD_BINARY_OP_EXTEND_RHS_r22] = _GUARD_BINARY_OP_EXTEND_RHS, + [_GUARD_BINARY_OP_EXTEND_RHS_r33] = _GUARD_BINARY_OP_EXTEND_RHS, + [_GUARD_BINARY_OP_EXTEND_r22] = _GUARD_BINARY_OP_EXTEND, + [_BINARY_OP_EXTEND_r23] = _BINARY_OP_EXTEND, + [_BINARY_SLICE_r31] = _BINARY_SLICE, + [_STORE_SLICE_r30] = _STORE_SLICE, + [_BINARY_OP_SUBSCR_LIST_INT_r23] = _BINARY_OP_SUBSCR_LIST_INT, + [_BINARY_OP_SUBSCR_LIST_SLICE_r23] = _BINARY_OP_SUBSCR_LIST_SLICE, + [_BINARY_OP_SUBSCR_STR_INT_r23] = _BINARY_OP_SUBSCR_STR_INT, + [_BINARY_OP_SUBSCR_USTR_INT_r23] = _BINARY_OP_SUBSCR_USTR_INT, + [_GUARD_NOS_TUPLE_r02] = _GUARD_NOS_TUPLE, + [_GUARD_NOS_TUPLE_r12] = _GUARD_NOS_TUPLE, + [_GUARD_NOS_TUPLE_r22] = _GUARD_NOS_TUPLE, + [_GUARD_NOS_TUPLE_r33] = _GUARD_NOS_TUPLE, + [_GUARD_TOS_TUPLE_r01] = _GUARD_TOS_TUPLE, + [_GUARD_TOS_TUPLE_r11] = _GUARD_TOS_TUPLE, + [_GUARD_TOS_TUPLE_r22] = _GUARD_TOS_TUPLE, + [_GUARD_TOS_TUPLE_r33] = _GUARD_TOS_TUPLE, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02] = _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12] = _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22] = _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33] = _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, + [_BINARY_OP_SUBSCR_TUPLE_INT_r03] = _BINARY_OP_SUBSCR_TUPLE_INT, + [_BINARY_OP_SUBSCR_TUPLE_INT_r13] = _BINARY_OP_SUBSCR_TUPLE_INT, + [_BINARY_OP_SUBSCR_TUPLE_INT_r23] = _BINARY_OP_SUBSCR_TUPLE_INT, + [_GUARD_NOS_DICT_r02] = _GUARD_NOS_DICT, + [_GUARD_NOS_DICT_r12] = _GUARD_NOS_DICT, + [_GUARD_NOS_DICT_r22] = _GUARD_NOS_DICT, + [_GUARD_NOS_DICT_r33] = _GUARD_NOS_DICT, + [_GUARD_NOS_ANY_DICT_r02] = _GUARD_NOS_ANY_DICT, + [_GUARD_NOS_ANY_DICT_r12] = _GUARD_NOS_ANY_DICT, + [_GUARD_NOS_ANY_DICT_r22] = _GUARD_NOS_ANY_DICT, + [_GUARD_NOS_ANY_DICT_r33] = _GUARD_NOS_ANY_DICT, + [_GUARD_TOS_ANY_DICT_r01] = _GUARD_TOS_ANY_DICT, + [_GUARD_TOS_ANY_DICT_r11] = _GUARD_TOS_ANY_DICT, + [_GUARD_TOS_ANY_DICT_r22] = _GUARD_TOS_ANY_DICT, + [_GUARD_TOS_ANY_DICT_r33] = _GUARD_TOS_ANY_DICT, + [_GUARD_TOS_DICT_r01] = _GUARD_TOS_DICT, + [_GUARD_TOS_DICT_r11] = _GUARD_TOS_DICT, + [_GUARD_TOS_DICT_r22] = _GUARD_TOS_DICT, + [_GUARD_TOS_DICT_r33] = _GUARD_TOS_DICT, + [_GUARD_TOS_FROZENDICT_r01] = _GUARD_TOS_FROZENDICT, + [_GUARD_TOS_FROZENDICT_r11] = _GUARD_TOS_FROZENDICT, + [_GUARD_TOS_FROZENDICT_r22] = _GUARD_TOS_FROZENDICT, + [_GUARD_TOS_FROZENDICT_r33] = _GUARD_TOS_FROZENDICT, + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23] = _BINARY_OP_SUBSCR_DICT_KNOWN_HASH, + [_BINARY_OP_SUBSCR_DICT_r23] = _BINARY_OP_SUBSCR_DICT, + [_BINARY_OP_SUBSCR_CHECK_FUNC_r23] = _BINARY_OP_SUBSCR_CHECK_FUNC, + [_BINARY_OP_SUBSCR_INIT_CALL_r01] = _BINARY_OP_SUBSCR_INIT_CALL, + [_BINARY_OP_SUBSCR_INIT_CALL_r11] = _BINARY_OP_SUBSCR_INIT_CALL, + [_BINARY_OP_SUBSCR_INIT_CALL_r21] = _BINARY_OP_SUBSCR_INIT_CALL, + [_BINARY_OP_SUBSCR_INIT_CALL_r31] = _BINARY_OP_SUBSCR_INIT_CALL, + [_LIST_APPEND_r10] = _LIST_APPEND, + [_SET_ADD_r10] = _SET_ADD, + [_STORE_SUBSCR_r30] = _STORE_SUBSCR, + [_STORE_SUBSCR_LIST_INT_r32] = _STORE_SUBSCR_LIST_INT, + [_STORE_SUBSCR_DICT_r31] = _STORE_SUBSCR_DICT, + [_STORE_SUBSCR_DICT_KNOWN_HASH_r31] = _STORE_SUBSCR_DICT_KNOWN_HASH, + [_DELETE_SUBSCR_r20] = _DELETE_SUBSCR, + [_CALL_INTRINSIC_1_r12] = _CALL_INTRINSIC_1, + [_CALL_INTRINSIC_2_r23] = _CALL_INTRINSIC_2, + [_MAKE_HEAP_SAFE_r01] = _MAKE_HEAP_SAFE, + [_MAKE_HEAP_SAFE_r11] = _MAKE_HEAP_SAFE, + [_MAKE_HEAP_SAFE_r22] = _MAKE_HEAP_SAFE, + [_MAKE_HEAP_SAFE_r33] = _MAKE_HEAP_SAFE, + [_RETURN_VALUE_r11] = _RETURN_VALUE, + [_GET_AITER_r11] = _GET_AITER, + [_GET_ANEXT_r12] = _GET_ANEXT, + [_GET_AWAITABLE_r11] = _GET_AWAITABLE, + [_SEND_GEN_FRAME_r33] = _SEND_GEN_FRAME, + [_YIELD_VALUE_r11] = _YIELD_VALUE, + [_POP_EXCEPT_r10] = _POP_EXCEPT, + [_LOAD_COMMON_CONSTANT_r01] = _LOAD_COMMON_CONSTANT, + [_LOAD_COMMON_CONSTANT_r12] = _LOAD_COMMON_CONSTANT, + [_LOAD_COMMON_CONSTANT_r23] = _LOAD_COMMON_CONSTANT, + [_LOAD_BUILD_CLASS_r01] = _LOAD_BUILD_CLASS, + [_STORE_NAME_r10] = _STORE_NAME, + [_DELETE_NAME_r00] = _DELETE_NAME, + [_UNPACK_SEQUENCE_r10] = _UNPACK_SEQUENCE, + [_UNPACK_SEQUENCE_TWO_TUPLE_r12] = _UNPACK_SEQUENCE_TWO_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02] = _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12] = _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23] = _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03] = _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13] = _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE, + [_UNPACK_SEQUENCE_TUPLE_r10] = _UNPACK_SEQUENCE_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_TUPLE_r10] = _UNPACK_SEQUENCE_UNIQUE_TUPLE, + [_UNPACK_SEQUENCE_LIST_r10] = _UNPACK_SEQUENCE_LIST, + [_UNPACK_EX_r10] = _UNPACK_EX, + [_STORE_ATTR_r20] = _STORE_ATTR, + [_DELETE_ATTR_r10] = _DELETE_ATTR, + [_STORE_GLOBAL_r10] = _STORE_GLOBAL, + [_DELETE_GLOBAL_r00] = _DELETE_GLOBAL, + [_LOAD_LOCALS_r01] = _LOAD_LOCALS, + [_LOAD_LOCALS_r12] = _LOAD_LOCALS, + [_LOAD_LOCALS_r23] = _LOAD_LOCALS, + [_LOAD_NAME_r01] = _LOAD_NAME, + [_LOAD_GLOBAL_r00] = _LOAD_GLOBAL, + [_PUSH_NULL_CONDITIONAL_r00] = _PUSH_NULL_CONDITIONAL, + [_GUARD_GLOBALS_VERSION_r00] = _GUARD_GLOBALS_VERSION, + [_GUARD_GLOBALS_VERSION_r11] = _GUARD_GLOBALS_VERSION, + [_GUARD_GLOBALS_VERSION_r22] = _GUARD_GLOBALS_VERSION, + [_GUARD_GLOBALS_VERSION_r33] = _GUARD_GLOBALS_VERSION, + [_LOAD_GLOBAL_MODULE_r01] = _LOAD_GLOBAL_MODULE, + [_LOAD_GLOBAL_BUILTINS_r01] = _LOAD_GLOBAL_BUILTINS, + [_DELETE_FAST_r00] = _DELETE_FAST, + [_MAKE_CELL_r00] = _MAKE_CELL, + [_DELETE_DEREF_r00] = _DELETE_DEREF, + [_LOAD_FROM_DICT_OR_DEREF_r11] = _LOAD_FROM_DICT_OR_DEREF, + [_LOAD_DEREF_r01] = _LOAD_DEREF, + [_STORE_DEREF_r10] = _STORE_DEREF, + [_COPY_FREE_VARS_r00] = _COPY_FREE_VARS, + [_COPY_FREE_VARS_r11] = _COPY_FREE_VARS, + [_COPY_FREE_VARS_r22] = _COPY_FREE_VARS, + [_COPY_FREE_VARS_r33] = _COPY_FREE_VARS, + [_BUILD_STRING_r01] = _BUILD_STRING, + [_BUILD_INTERPOLATION_r01] = _BUILD_INTERPOLATION, + [_BUILD_TEMPLATE_r21] = _BUILD_TEMPLATE, + [_BUILD_TUPLE_r01] = _BUILD_TUPLE, + [_BUILD_LIST_r01] = _BUILD_LIST, + [_LIST_EXTEND_r11] = _LIST_EXTEND, + [_SET_UPDATE_r11] = _SET_UPDATE, + [_BUILD_SET_r01] = _BUILD_SET, + [_BUILD_MAP_r01] = _BUILD_MAP, + [_SETUP_ANNOTATIONS_r00] = _SETUP_ANNOTATIONS, + [_DICT_UPDATE_r11] = _DICT_UPDATE, + [_DICT_MERGE_r11] = _DICT_MERGE, + [_MAP_ADD_r20] = _MAP_ADD, + [_LOAD_SUPER_ATTR_ATTR_r31] = _LOAD_SUPER_ATTR_ATTR, + [_GUARD_NOS_TYPE_VERSION_r02] = _GUARD_NOS_TYPE_VERSION, + [_GUARD_NOS_TYPE_VERSION_r12] = _GUARD_NOS_TYPE_VERSION, + [_GUARD_NOS_TYPE_VERSION_r22] = _GUARD_NOS_TYPE_VERSION, + [_GUARD_NOS_TYPE_VERSION_r33] = _GUARD_NOS_TYPE_VERSION, + [_GUARD_LOAD_SUPER_ATTR_METHOD_r03] = _GUARD_LOAD_SUPER_ATTR_METHOD, + [_GUARD_LOAD_SUPER_ATTR_METHOD_r13] = _GUARD_LOAD_SUPER_ATTR_METHOD, + [_GUARD_LOAD_SUPER_ATTR_METHOD_r23] = _GUARD_LOAD_SUPER_ATTR_METHOD, + [_GUARD_LOAD_SUPER_ATTR_METHOD_r33] = _GUARD_LOAD_SUPER_ATTR_METHOD, + [_LOAD_SUPER_ATTR_METHOD_r32] = _LOAD_SUPER_ATTR_METHOD, + [_LOAD_ATTR_r10] = _LOAD_ATTR, + [_GUARD_TYPE_VERSION_r01] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_r11] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_r22] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_r33] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_LOCKED_r01] = _GUARD_TYPE_VERSION_LOCKED, + [_GUARD_TYPE_VERSION_LOCKED_r11] = _GUARD_TYPE_VERSION_LOCKED, + [_GUARD_TYPE_VERSION_LOCKED_r22] = _GUARD_TYPE_VERSION_LOCKED, + [_GUARD_TYPE_VERSION_LOCKED_r33] = _GUARD_TYPE_VERSION_LOCKED, + [_GUARD_TYPE_r01] = _GUARD_TYPE, + [_GUARD_TYPE_r11] = _GUARD_TYPE, + [_GUARD_TYPE_r22] = _GUARD_TYPE, + [_GUARD_TYPE_r33] = _GUARD_TYPE, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r01] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r11] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r22] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r33] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_LOAD_ATTR_INSTANCE_VALUE_r02] = _LOAD_ATTR_INSTANCE_VALUE, + [_LOAD_ATTR_INSTANCE_VALUE_r12] = _LOAD_ATTR_INSTANCE_VALUE, + [_LOAD_ATTR_INSTANCE_VALUE_r23] = _LOAD_ATTR_INSTANCE_VALUE, + [_LOAD_ATTR_MODULE_r12] = _LOAD_ATTR_MODULE, + [_LOAD_ATTR_WITH_HINT_r12] = _LOAD_ATTR_WITH_HINT, + [_LOAD_ATTR_SLOT_r02] = _LOAD_ATTR_SLOT, + [_LOAD_ATTR_SLOT_r12] = _LOAD_ATTR_SLOT, + [_LOAD_ATTR_SLOT_r23] = _LOAD_ATTR_SLOT, + [_CHECK_ATTR_CLASS_r01] = _CHECK_ATTR_CLASS, + [_CHECK_ATTR_CLASS_r11] = _CHECK_ATTR_CLASS, + [_CHECK_ATTR_CLASS_r22] = _CHECK_ATTR_CLASS, + [_CHECK_ATTR_CLASS_r33] = _CHECK_ATTR_CLASS, + [_LOAD_ATTR_CLASS_r11] = _LOAD_ATTR_CLASS, + [_LOAD_ATTR_PROPERTY_FRAME_r01] = _LOAD_ATTR_PROPERTY_FRAME, + [_LOAD_ATTR_PROPERTY_FRAME_r11] = _LOAD_ATTR_PROPERTY_FRAME, + [_LOAD_ATTR_PROPERTY_FRAME_r22] = _LOAD_ATTR_PROPERTY_FRAME, + [_LOAD_ATTR_PROPERTY_FRAME_r33] = _LOAD_ATTR_PROPERTY_FRAME, + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11] = _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME, + [_GUARD_DORV_NO_DICT_r01] = _GUARD_DORV_NO_DICT, + [_GUARD_DORV_NO_DICT_r11] = _GUARD_DORV_NO_DICT, + [_GUARD_DORV_NO_DICT_r22] = _GUARD_DORV_NO_DICT, + [_GUARD_DORV_NO_DICT_r33] = _GUARD_DORV_NO_DICT, + [_STORE_ATTR_INSTANCE_VALUE_r21] = _STORE_ATTR_INSTANCE_VALUE, + [_LOCK_OBJECT_r01] = _LOCK_OBJECT, + [_LOCK_OBJECT_r11] = _LOCK_OBJECT, + [_LOCK_OBJECT_r22] = _LOCK_OBJECT, + [_LOCK_OBJECT_r33] = _LOCK_OBJECT, + [_STORE_ATTR_WITH_HINT_r21] = _STORE_ATTR_WITH_HINT, + [_STORE_ATTR_SLOT_r21] = _STORE_ATTR_SLOT, + [_COMPARE_OP_r21] = _COMPARE_OP, + [_COMPARE_OP_FLOAT_r03] = _COMPARE_OP_FLOAT, + [_COMPARE_OP_FLOAT_r13] = _COMPARE_OP_FLOAT, + [_COMPARE_OP_FLOAT_r23] = _COMPARE_OP_FLOAT, + [_COMPARE_OP_INT_r23] = _COMPARE_OP_INT, + [_COMPARE_OP_STR_r23] = _COMPARE_OP_STR, + [_IS_OP_r03] = _IS_OP, + [_IS_OP_r13] = _IS_OP, + [_IS_OP_r23] = _IS_OP, + [_CONTAINS_OP_r23] = _CONTAINS_OP, + [_GUARD_TOS_ANY_SET_r01] = _GUARD_TOS_ANY_SET, + [_GUARD_TOS_ANY_SET_r11] = _GUARD_TOS_ANY_SET, + [_GUARD_TOS_ANY_SET_r22] = _GUARD_TOS_ANY_SET, + [_GUARD_TOS_ANY_SET_r33] = _GUARD_TOS_ANY_SET, + [_GUARD_TOS_SET_r01] = _GUARD_TOS_SET, + [_GUARD_TOS_SET_r11] = _GUARD_TOS_SET, + [_GUARD_TOS_SET_r22] = _GUARD_TOS_SET, + [_GUARD_TOS_SET_r33] = _GUARD_TOS_SET, + [_GUARD_TOS_FROZENSET_r01] = _GUARD_TOS_FROZENSET, + [_GUARD_TOS_FROZENSET_r11] = _GUARD_TOS_FROZENSET, + [_GUARD_TOS_FROZENSET_r22] = _GUARD_TOS_FROZENSET, + [_GUARD_TOS_FROZENSET_r33] = _GUARD_TOS_FROZENSET, + [_CONTAINS_OP_SET_r23] = _CONTAINS_OP_SET, + [_CONTAINS_OP_DICT_r23] = _CONTAINS_OP_DICT, + [_CHECK_EG_MATCH_r22] = _CHECK_EG_MATCH, + [_CHECK_EXC_MATCH_r22] = _CHECK_EXC_MATCH, + [_IMPORT_NAME_r21] = _IMPORT_NAME, + [_IMPORT_FROM_r12] = _IMPORT_FROM, + [_IS_NONE_r11] = _IS_NONE, + [_GET_LEN_r12] = _GET_LEN, + [_MATCH_CLASS_r33] = _MATCH_CLASS, + [_MATCH_MAPPING_r02] = _MATCH_MAPPING, + [_MATCH_MAPPING_r12] = _MATCH_MAPPING, + [_MATCH_MAPPING_r23] = _MATCH_MAPPING, + [_MATCH_SEQUENCE_r02] = _MATCH_SEQUENCE, + [_MATCH_SEQUENCE_r12] = _MATCH_SEQUENCE, + [_MATCH_SEQUENCE_r23] = _MATCH_SEQUENCE, + [_MATCH_KEYS_r23] = _MATCH_KEYS, + [_GET_ITER_r12] = _GET_ITER, + [_GUARD_ITERATOR_r01] = _GUARD_ITERATOR, + [_GUARD_ITERATOR_r11] = _GUARD_ITERATOR, + [_GUARD_ITERATOR_r22] = _GUARD_ITERATOR, + [_GUARD_ITERATOR_r33] = _GUARD_ITERATOR, + [_GUARD_ITER_VIRTUAL_r01] = _GUARD_ITER_VIRTUAL, + [_GUARD_ITER_VIRTUAL_r11] = _GUARD_ITER_VIRTUAL, + [_GUARD_ITER_VIRTUAL_r22] = _GUARD_ITER_VIRTUAL, + [_GUARD_ITER_VIRTUAL_r33] = _GUARD_ITER_VIRTUAL, + [_PUSH_TAGGED_ZERO_r01] = _PUSH_TAGGED_ZERO, + [_PUSH_TAGGED_ZERO_r12] = _PUSH_TAGGED_ZERO, + [_PUSH_TAGGED_ZERO_r23] = _PUSH_TAGGED_ZERO, + [_GET_ITER_TRAD_r12] = _GET_ITER_TRAD, + [_FOR_ITER_TIER_TWO_r23] = _FOR_ITER_TIER_TWO, + [_GUARD_NOS_ITER_VIRTUAL_r02] = _GUARD_NOS_ITER_VIRTUAL, + [_GUARD_NOS_ITER_VIRTUAL_r12] = _GUARD_NOS_ITER_VIRTUAL, + [_GUARD_NOS_ITER_VIRTUAL_r22] = _GUARD_NOS_ITER_VIRTUAL, + [_GUARD_NOS_ITER_VIRTUAL_r33] = _GUARD_NOS_ITER_VIRTUAL, + [_FOR_ITER_VIRTUAL_TIER_TWO_r23] = _FOR_ITER_VIRTUAL_TIER_TWO, + [_ITER_CHECK_LIST_r02] = _ITER_CHECK_LIST, + [_ITER_CHECK_LIST_r12] = _ITER_CHECK_LIST, + [_ITER_CHECK_LIST_r22] = _ITER_CHECK_LIST, + [_ITER_CHECK_LIST_r33] = _ITER_CHECK_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r02] = _GUARD_NOT_EXHAUSTED_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r12] = _GUARD_NOT_EXHAUSTED_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r22] = _GUARD_NOT_EXHAUSTED_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r33] = _GUARD_NOT_EXHAUSTED_LIST, + [_ITER_NEXT_LIST_TIER_TWO_r23] = _ITER_NEXT_LIST_TIER_TWO, + [_ITER_CHECK_TUPLE_r02] = _ITER_CHECK_TUPLE, + [_ITER_CHECK_TUPLE_r12] = _ITER_CHECK_TUPLE, + [_ITER_CHECK_TUPLE_r22] = _ITER_CHECK_TUPLE, + [_ITER_CHECK_TUPLE_r33] = _ITER_CHECK_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r02] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r12] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r22] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r33] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_ITER_NEXT_TUPLE_r03] = _ITER_NEXT_TUPLE, + [_ITER_NEXT_TUPLE_r13] = _ITER_NEXT_TUPLE, + [_ITER_NEXT_TUPLE_r23] = _ITER_NEXT_TUPLE, + [_ITER_CHECK_RANGE_r02] = _ITER_CHECK_RANGE, + [_ITER_CHECK_RANGE_r12] = _ITER_CHECK_RANGE, + [_ITER_CHECK_RANGE_r22] = _ITER_CHECK_RANGE, + [_ITER_CHECK_RANGE_r33] = _ITER_CHECK_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r02] = _GUARD_NOT_EXHAUSTED_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r12] = _GUARD_NOT_EXHAUSTED_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r22] = _GUARD_NOT_EXHAUSTED_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r33] = _GUARD_NOT_EXHAUSTED_RANGE, + [_ITER_NEXT_RANGE_r03] = _ITER_NEXT_RANGE, + [_ITER_NEXT_RANGE_r13] = _ITER_NEXT_RANGE, + [_ITER_NEXT_RANGE_r23] = _ITER_NEXT_RANGE, + [_FOR_ITER_GEN_FRAME_r03] = _FOR_ITER_GEN_FRAME, + [_FOR_ITER_GEN_FRAME_r13] = _FOR_ITER_GEN_FRAME, + [_FOR_ITER_GEN_FRAME_r23] = _FOR_ITER_GEN_FRAME, + [_INSERT_NULL_r10] = _INSERT_NULL, + [_LOAD_SPECIAL_r00] = _LOAD_SPECIAL, + [_WITH_EXCEPT_START_r33] = _WITH_EXCEPT_START, + [_PUSH_EXC_INFO_r02] = _PUSH_EXC_INFO, + [_PUSH_EXC_INFO_r12] = _PUSH_EXC_INFO, + [_PUSH_EXC_INFO_r23] = _PUSH_EXC_INFO, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, + [_GUARD_KEYS_VERSION_r01] = _GUARD_KEYS_VERSION, + [_GUARD_KEYS_VERSION_r11] = _GUARD_KEYS_VERSION, + [_GUARD_KEYS_VERSION_r22] = _GUARD_KEYS_VERSION, + [_GUARD_KEYS_VERSION_r33] = _GUARD_KEYS_VERSION, + [_LOAD_ATTR_METHOD_WITH_VALUES_r02] = _LOAD_ATTR_METHOD_WITH_VALUES, + [_LOAD_ATTR_METHOD_WITH_VALUES_r12] = _LOAD_ATTR_METHOD_WITH_VALUES, + [_LOAD_ATTR_METHOD_WITH_VALUES_r23] = _LOAD_ATTR_METHOD_WITH_VALUES, + [_LOAD_ATTR_METHOD_NO_DICT_r02] = _LOAD_ATTR_METHOD_NO_DICT, + [_LOAD_ATTR_METHOD_NO_DICT_r12] = _LOAD_ATTR_METHOD_NO_DICT, + [_LOAD_ATTR_METHOD_NO_DICT_r23] = _LOAD_ATTR_METHOD_NO_DICT, + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11] = _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11] = _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r01] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r11] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r22] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r33] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_LOAD_ATTR_METHOD_LAZY_DICT_r02] = _LOAD_ATTR_METHOD_LAZY_DICT, + [_LOAD_ATTR_METHOD_LAZY_DICT_r12] = _LOAD_ATTR_METHOD_LAZY_DICT, + [_LOAD_ATTR_METHOD_LAZY_DICT_r23] = _LOAD_ATTR_METHOD_LAZY_DICT, + [_MAYBE_EXPAND_METHOD_r00] = _MAYBE_EXPAND_METHOD, + [_PY_FRAME_GENERAL_r01] = _PY_FRAME_GENERAL, + [_CHECK_FUNCTION_VERSION_r00] = _CHECK_FUNCTION_VERSION, + [_CHECK_FUNCTION_VERSION_INLINE_r00] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_FUNCTION_VERSION_INLINE_r11] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_FUNCTION_VERSION_INLINE_r22] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_FUNCTION_VERSION_INLINE_r33] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_METHOD_VERSION_r00] = _CHECK_METHOD_VERSION, + [_EXPAND_METHOD_r00] = _EXPAND_METHOD, + [_CHECK_IS_NOT_PY_CALLABLE_r00] = _CHECK_IS_NOT_PY_CALLABLE, + [_CALL_NON_PY_GENERAL_r01] = _CALL_NON_PY_GENERAL, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00] = _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00] = _INIT_CALL_BOUND_METHOD_EXACT_ARGS, + [_CHECK_PEP_523_r00] = _CHECK_PEP_523, + [_CHECK_PEP_523_r11] = _CHECK_PEP_523, + [_CHECK_PEP_523_r22] = _CHECK_PEP_523, + [_CHECK_PEP_523_r33] = _CHECK_PEP_523, + [_CHECK_FUNCTION_EXACT_ARGS_r00] = _CHECK_FUNCTION_EXACT_ARGS, + [_CHECK_STACK_SPACE_r00] = _CHECK_STACK_SPACE, + [_CHECK_RECURSION_REMAINING_r00] = _CHECK_RECURSION_REMAINING, + [_CHECK_RECURSION_REMAINING_r11] = _CHECK_RECURSION_REMAINING, + [_CHECK_RECURSION_REMAINING_r22] = _CHECK_RECURSION_REMAINING, + [_CHECK_RECURSION_REMAINING_r33] = _CHECK_RECURSION_REMAINING, + [_INIT_CALL_PY_EXACT_ARGS_0_r01] = _INIT_CALL_PY_EXACT_ARGS_0, + [_INIT_CALL_PY_EXACT_ARGS_1_r01] = _INIT_CALL_PY_EXACT_ARGS_1, + [_INIT_CALL_PY_EXACT_ARGS_2_r01] = _INIT_CALL_PY_EXACT_ARGS_2, + [_INIT_CALL_PY_EXACT_ARGS_3_r01] = _INIT_CALL_PY_EXACT_ARGS_3, + [_INIT_CALL_PY_EXACT_ARGS_4_r01] = _INIT_CALL_PY_EXACT_ARGS_4, + [_INIT_CALL_PY_EXACT_ARGS_r01] = _INIT_CALL_PY_EXACT_ARGS, + [_PUSH_FRAME_r10] = _PUSH_FRAME, + [_GUARD_NOS_NULL_r02] = _GUARD_NOS_NULL, + [_GUARD_NOS_NULL_r12] = _GUARD_NOS_NULL, + [_GUARD_NOS_NULL_r22] = _GUARD_NOS_NULL, + [_GUARD_NOS_NULL_r33] = _GUARD_NOS_NULL, + [_GUARD_NOS_NOT_NULL_r02] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r12] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r22] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r33] = _GUARD_NOS_NOT_NULL, + [_GUARD_THIRD_NULL_r03] = _GUARD_THIRD_NULL, + [_GUARD_THIRD_NULL_r13] = _GUARD_THIRD_NULL, + [_GUARD_THIRD_NULL_r23] = _GUARD_THIRD_NULL, + [_GUARD_THIRD_NULL_r33] = _GUARD_THIRD_NULL, + [_GUARD_CALLABLE_TYPE_1_r03] = _GUARD_CALLABLE_TYPE_1, + [_GUARD_CALLABLE_TYPE_1_r13] = _GUARD_CALLABLE_TYPE_1, + [_GUARD_CALLABLE_TYPE_1_r23] = _GUARD_CALLABLE_TYPE_1, + [_GUARD_CALLABLE_TYPE_1_r33] = _GUARD_CALLABLE_TYPE_1, + [_CALL_TYPE_1_r02] = _CALL_TYPE_1, + [_CALL_TYPE_1_r12] = _CALL_TYPE_1, + [_CALL_TYPE_1_r22] = _CALL_TYPE_1, + [_CALL_TYPE_1_r32] = _CALL_TYPE_1, + [_GUARD_CALLABLE_STR_1_r03] = _GUARD_CALLABLE_STR_1, + [_GUARD_CALLABLE_STR_1_r13] = _GUARD_CALLABLE_STR_1, + [_GUARD_CALLABLE_STR_1_r23] = _GUARD_CALLABLE_STR_1, + [_GUARD_CALLABLE_STR_1_r33] = _GUARD_CALLABLE_STR_1, + [_CALL_STR_1_r32] = _CALL_STR_1, + [_GUARD_CALLABLE_TUPLE_1_r03] = _GUARD_CALLABLE_TUPLE_1, + [_GUARD_CALLABLE_TUPLE_1_r13] = _GUARD_CALLABLE_TUPLE_1, + [_GUARD_CALLABLE_TUPLE_1_r23] = _GUARD_CALLABLE_TUPLE_1, + [_GUARD_CALLABLE_TUPLE_1_r33] = _GUARD_CALLABLE_TUPLE_1, + [_CALL_TUPLE_1_r32] = _CALL_TUPLE_1, + [_CHECK_OBJECT_r00] = _CHECK_OBJECT, + [_ALLOCATE_OBJECT_r00] = _ALLOCATE_OBJECT, + [_CREATE_INIT_FRAME_r01] = _CREATE_INIT_FRAME, + [_EXIT_INIT_CHECK_r10] = _EXIT_INIT_CHECK, + [_GUARD_CALLABLE_BUILTIN_CLASS_r00] = _GUARD_CALLABLE_BUILTIN_CLASS, + [_CALL_BUILTIN_CLASS_r00] = _CALL_BUILTIN_CLASS, + [_GUARD_CALLABLE_BUILTIN_O_r00] = _GUARD_CALLABLE_BUILTIN_O, + [_CALL_BUILTIN_O_r03] = _CALL_BUILTIN_O, + [_GUARD_CALLABLE_BUILTIN_FAST_r00] = _GUARD_CALLABLE_BUILTIN_FAST, + [_CALL_BUILTIN_FAST_r00] = _CALL_BUILTIN_FAST, + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00] = _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS_r00] = _CALL_BUILTIN_FAST_WITH_KEYWORDS, + [_GUARD_CALLABLE_LEN_r03] = _GUARD_CALLABLE_LEN, + [_GUARD_CALLABLE_LEN_r13] = _GUARD_CALLABLE_LEN, + [_GUARD_CALLABLE_LEN_r23] = _GUARD_CALLABLE_LEN, + [_GUARD_CALLABLE_LEN_r33] = _GUARD_CALLABLE_LEN, + [_CALL_LEN_r33] = _CALL_LEN, + [_GUARD_CALLABLE_ISINSTANCE_r03] = _GUARD_CALLABLE_ISINSTANCE, + [_GUARD_CALLABLE_ISINSTANCE_r13] = _GUARD_CALLABLE_ISINSTANCE, + [_GUARD_CALLABLE_ISINSTANCE_r23] = _GUARD_CALLABLE_ISINSTANCE, + [_GUARD_CALLABLE_ISINSTANCE_r33] = _GUARD_CALLABLE_ISINSTANCE, + [_CALL_ISINSTANCE_r31] = _CALL_ISINSTANCE, + [_GUARD_CALLABLE_LIST_APPEND_r03] = _GUARD_CALLABLE_LIST_APPEND, + [_GUARD_CALLABLE_LIST_APPEND_r13] = _GUARD_CALLABLE_LIST_APPEND, + [_GUARD_CALLABLE_LIST_APPEND_r23] = _GUARD_CALLABLE_LIST_APPEND, + [_GUARD_CALLABLE_LIST_APPEND_r33] = _GUARD_CALLABLE_LIST_APPEND, + [_CALL_LIST_APPEND_r03] = _CALL_LIST_APPEND, + [_CALL_LIST_APPEND_r13] = _CALL_LIST_APPEND, + [_CALL_LIST_APPEND_r23] = _CALL_LIST_APPEND, + [_CALL_LIST_APPEND_r33] = _CALL_LIST_APPEND, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_O, + [_CALL_METHOD_DESCRIPTOR_O_r03] = _CALL_METHOD_DESCRIPTOR_O, + [_CHECK_RECURSION_LIMIT_r00] = _CHECK_RECURSION_LIMIT, + [_CHECK_RECURSION_LIMIT_r11] = _CHECK_RECURSION_LIMIT, + [_CHECK_RECURSION_LIMIT_r22] = _CHECK_RECURSION_LIMIT, + [_CHECK_RECURSION_LIMIT_r33] = _CHECK_RECURSION_LIMIT, + [_CALL_METHOD_DESCRIPTOR_O_INLINE_r03] = _CALL_METHOD_DESCRIPTOR_O_INLINE, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00] = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, + [_CALL_METHOD_DESCRIPTOR_NOARGS_r03] = _CALL_METHOD_DESCRIPTOR_NOARGS, + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03] = _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, + [_CALL_METHOD_DESCRIPTOR_FAST_r00] = _CALL_METHOD_DESCRIPTOR_FAST, + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00] = _CALL_METHOD_DESCRIPTOR_FAST_INLINE, + [_MAYBE_EXPAND_METHOD_KW_r11] = _MAYBE_EXPAND_METHOD_KW, + [_PY_FRAME_KW_r11] = _PY_FRAME_KW, + [_CHECK_FUNCTION_VERSION_KW_r11] = _CHECK_FUNCTION_VERSION_KW, + [_CHECK_METHOD_VERSION_KW_r11] = _CHECK_METHOD_VERSION_KW, + [_EXPAND_METHOD_KW_r11] = _EXPAND_METHOD_KW, + [_CHECK_IS_NOT_PY_CALLABLE_KW_r11] = _CHECK_IS_NOT_PY_CALLABLE_KW, + [_CALL_KW_NON_PY_r11] = _CALL_KW_NON_PY, + [_MAKE_CALLARGS_A_TUPLE_r33] = _MAKE_CALLARGS_A_TUPLE, + [_CHECK_IS_PY_CALLABLE_EX_r03] = _CHECK_IS_PY_CALLABLE_EX, + [_CHECK_IS_PY_CALLABLE_EX_r13] = _CHECK_IS_PY_CALLABLE_EX, + [_CHECK_IS_PY_CALLABLE_EX_r23] = _CHECK_IS_PY_CALLABLE_EX, + [_CHECK_IS_PY_CALLABLE_EX_r33] = _CHECK_IS_PY_CALLABLE_EX, + [_PY_FRAME_EX_r31] = _PY_FRAME_EX, + [_CHECK_IS_NOT_PY_CALLABLE_EX_r03] = _CHECK_IS_NOT_PY_CALLABLE_EX, + [_CHECK_IS_NOT_PY_CALLABLE_EX_r13] = _CHECK_IS_NOT_PY_CALLABLE_EX, + [_CHECK_IS_NOT_PY_CALLABLE_EX_r23] = _CHECK_IS_NOT_PY_CALLABLE_EX, + [_CHECK_IS_NOT_PY_CALLABLE_EX_r33] = _CHECK_IS_NOT_PY_CALLABLE_EX, + [_CALL_FUNCTION_EX_NON_PY_GENERAL_r31] = _CALL_FUNCTION_EX_NON_PY_GENERAL, + [_MAKE_FUNCTION_r12] = _MAKE_FUNCTION, + [_SET_FUNCTION_ATTRIBUTE_r01] = _SET_FUNCTION_ATTRIBUTE, + [_SET_FUNCTION_ATTRIBUTE_r11] = _SET_FUNCTION_ATTRIBUTE, + [_SET_FUNCTION_ATTRIBUTE_r21] = _SET_FUNCTION_ATTRIBUTE, + [_SET_FUNCTION_ATTRIBUTE_r32] = _SET_FUNCTION_ATTRIBUTE, + [_RETURN_GENERATOR_r01] = _RETURN_GENERATOR, + [_BUILD_SLICE_r01] = _BUILD_SLICE, + [_CONVERT_VALUE_r11] = _CONVERT_VALUE, + [_FORMAT_SIMPLE_r11] = _FORMAT_SIMPLE, + [_FORMAT_WITH_SPEC_r21] = _FORMAT_WITH_SPEC, + [_COPY_1_r02] = _COPY_1, + [_COPY_1_r12] = _COPY_1, + [_COPY_1_r23] = _COPY_1, + [_COPY_2_r03] = _COPY_2, + [_COPY_2_r13] = _COPY_2, + [_COPY_2_r23] = _COPY_2, + [_COPY_3_r03] = _COPY_3, + [_COPY_3_r13] = _COPY_3, + [_COPY_3_r23] = _COPY_3, + [_COPY_3_r33] = _COPY_3, + [_COPY_r01] = _COPY, + [_BINARY_OP_r23] = _BINARY_OP, + [_SWAP_2_r02] = _SWAP_2, + [_SWAP_2_r12] = _SWAP_2, + [_SWAP_2_r22] = _SWAP_2, + [_SWAP_2_r33] = _SWAP_2, + [_SWAP_3_r03] = _SWAP_3, + [_SWAP_3_r13] = _SWAP_3, + [_SWAP_3_r23] = _SWAP_3, + [_SWAP_3_r33] = _SWAP_3, + [_SWAP_r11] = _SWAP, + [_GUARD_IS_TRUE_POP_r00] = _GUARD_IS_TRUE_POP, + [_GUARD_IS_TRUE_POP_r10] = _GUARD_IS_TRUE_POP, + [_GUARD_IS_TRUE_POP_r21] = _GUARD_IS_TRUE_POP, + [_GUARD_IS_TRUE_POP_r32] = _GUARD_IS_TRUE_POP, + [_GUARD_IS_FALSE_POP_r00] = _GUARD_IS_FALSE_POP, + [_GUARD_IS_FALSE_POP_r10] = _GUARD_IS_FALSE_POP, + [_GUARD_IS_FALSE_POP_r21] = _GUARD_IS_FALSE_POP, + [_GUARD_IS_FALSE_POP_r32] = _GUARD_IS_FALSE_POP, + [_GUARD_BIT_IS_SET_POP_4_r00] = _GUARD_BIT_IS_SET_POP_4, + [_GUARD_BIT_IS_SET_POP_4_r10] = _GUARD_BIT_IS_SET_POP_4, + [_GUARD_BIT_IS_SET_POP_4_r21] = _GUARD_BIT_IS_SET_POP_4, + [_GUARD_BIT_IS_SET_POP_4_r32] = _GUARD_BIT_IS_SET_POP_4, + [_GUARD_BIT_IS_SET_POP_5_r00] = _GUARD_BIT_IS_SET_POP_5, + [_GUARD_BIT_IS_SET_POP_5_r10] = _GUARD_BIT_IS_SET_POP_5, + [_GUARD_BIT_IS_SET_POP_5_r21] = _GUARD_BIT_IS_SET_POP_5, + [_GUARD_BIT_IS_SET_POP_5_r32] = _GUARD_BIT_IS_SET_POP_5, + [_GUARD_BIT_IS_SET_POP_6_r00] = _GUARD_BIT_IS_SET_POP_6, + [_GUARD_BIT_IS_SET_POP_6_r10] = _GUARD_BIT_IS_SET_POP_6, + [_GUARD_BIT_IS_SET_POP_6_r21] = _GUARD_BIT_IS_SET_POP_6, + [_GUARD_BIT_IS_SET_POP_6_r32] = _GUARD_BIT_IS_SET_POP_6, + [_GUARD_BIT_IS_SET_POP_7_r00] = _GUARD_BIT_IS_SET_POP_7, + [_GUARD_BIT_IS_SET_POP_7_r10] = _GUARD_BIT_IS_SET_POP_7, + [_GUARD_BIT_IS_SET_POP_7_r21] = _GUARD_BIT_IS_SET_POP_7, + [_GUARD_BIT_IS_SET_POP_7_r32] = _GUARD_BIT_IS_SET_POP_7, + [_GUARD_BIT_IS_SET_POP_r00] = _GUARD_BIT_IS_SET_POP, + [_GUARD_BIT_IS_SET_POP_r10] = _GUARD_BIT_IS_SET_POP, + [_GUARD_BIT_IS_SET_POP_r21] = _GUARD_BIT_IS_SET_POP, + [_GUARD_BIT_IS_SET_POP_r32] = _GUARD_BIT_IS_SET_POP, + [_GUARD_BIT_IS_UNSET_POP_4_r00] = _GUARD_BIT_IS_UNSET_POP_4, + [_GUARD_BIT_IS_UNSET_POP_4_r10] = _GUARD_BIT_IS_UNSET_POP_4, + [_GUARD_BIT_IS_UNSET_POP_4_r21] = _GUARD_BIT_IS_UNSET_POP_4, + [_GUARD_BIT_IS_UNSET_POP_4_r32] = _GUARD_BIT_IS_UNSET_POP_4, + [_GUARD_BIT_IS_UNSET_POP_5_r00] = _GUARD_BIT_IS_UNSET_POP_5, + [_GUARD_BIT_IS_UNSET_POP_5_r10] = _GUARD_BIT_IS_UNSET_POP_5, + [_GUARD_BIT_IS_UNSET_POP_5_r21] = _GUARD_BIT_IS_UNSET_POP_5, + [_GUARD_BIT_IS_UNSET_POP_5_r32] = _GUARD_BIT_IS_UNSET_POP_5, + [_GUARD_BIT_IS_UNSET_POP_6_r00] = _GUARD_BIT_IS_UNSET_POP_6, + [_GUARD_BIT_IS_UNSET_POP_6_r10] = _GUARD_BIT_IS_UNSET_POP_6, + [_GUARD_BIT_IS_UNSET_POP_6_r21] = _GUARD_BIT_IS_UNSET_POP_6, + [_GUARD_BIT_IS_UNSET_POP_6_r32] = _GUARD_BIT_IS_UNSET_POP_6, + [_GUARD_BIT_IS_UNSET_POP_7_r00] = _GUARD_BIT_IS_UNSET_POP_7, + [_GUARD_BIT_IS_UNSET_POP_7_r10] = _GUARD_BIT_IS_UNSET_POP_7, + [_GUARD_BIT_IS_UNSET_POP_7_r21] = _GUARD_BIT_IS_UNSET_POP_7, + [_GUARD_BIT_IS_UNSET_POP_7_r32] = _GUARD_BIT_IS_UNSET_POP_7, + [_GUARD_BIT_IS_UNSET_POP_r00] = _GUARD_BIT_IS_UNSET_POP, + [_GUARD_BIT_IS_UNSET_POP_r10] = _GUARD_BIT_IS_UNSET_POP, + [_GUARD_BIT_IS_UNSET_POP_r21] = _GUARD_BIT_IS_UNSET_POP, + [_GUARD_BIT_IS_UNSET_POP_r32] = _GUARD_BIT_IS_UNSET_POP, + [_GUARD_IS_NONE_POP_r00] = _GUARD_IS_NONE_POP, + [_GUARD_IS_NONE_POP_r10] = _GUARD_IS_NONE_POP, + [_GUARD_IS_NONE_POP_r21] = _GUARD_IS_NONE_POP, + [_GUARD_IS_NONE_POP_r32] = _GUARD_IS_NONE_POP, + [_GUARD_IS_NOT_NONE_POP_r10] = _GUARD_IS_NOT_NONE_POP, + [_JUMP_TO_TOP_r00] = _JUMP_TO_TOP, + [_SET_IP_r00] = _SET_IP, + [_SET_IP_r11] = _SET_IP, + [_SET_IP_r22] = _SET_IP, + [_SET_IP_r33] = _SET_IP, + [_CHECK_STACK_SPACE_OPERAND_r00] = _CHECK_STACK_SPACE_OPERAND, + [_CHECK_STACK_SPACE_OPERAND_r11] = _CHECK_STACK_SPACE_OPERAND, + [_CHECK_STACK_SPACE_OPERAND_r22] = _CHECK_STACK_SPACE_OPERAND, + [_CHECK_STACK_SPACE_OPERAND_r33] = _CHECK_STACK_SPACE_OPERAND, + [_SAVE_RETURN_OFFSET_r00] = _SAVE_RETURN_OFFSET, + [_SAVE_RETURN_OFFSET_r11] = _SAVE_RETURN_OFFSET, + [_SAVE_RETURN_OFFSET_r22] = _SAVE_RETURN_OFFSET, + [_SAVE_RETURN_OFFSET_r33] = _SAVE_RETURN_OFFSET, + [_EXIT_TRACE_r00] = _EXIT_TRACE, + [_EXIT_TRACE_r10] = _EXIT_TRACE, + [_EXIT_TRACE_r20] = _EXIT_TRACE, + [_EXIT_TRACE_r30] = _EXIT_TRACE, + [_DYNAMIC_EXIT_r00] = _DYNAMIC_EXIT, + [_DYNAMIC_EXIT_r10] = _DYNAMIC_EXIT, + [_DYNAMIC_EXIT_r20] = _DYNAMIC_EXIT, + [_DYNAMIC_EXIT_r30] = _DYNAMIC_EXIT, + [_CHECK_VALIDITY_r00] = _CHECK_VALIDITY, + [_CHECK_VALIDITY_r11] = _CHECK_VALIDITY, + [_CHECK_VALIDITY_r22] = _CHECK_VALIDITY, + [_CHECK_VALIDITY_r33] = _CHECK_VALIDITY, + [_LOAD_CONST_INLINE_r01] = _LOAD_CONST_INLINE, + [_LOAD_CONST_INLINE_r12] = _LOAD_CONST_INLINE, + [_LOAD_CONST_INLINE_r23] = _LOAD_CONST_INLINE, + [_LOAD_CONST_INLINE_BORROW_r01] = _LOAD_CONST_INLINE_BORROW, + [_LOAD_CONST_INLINE_BORROW_r12] = _LOAD_CONST_INLINE_BORROW, + [_LOAD_CONST_INLINE_BORROW_r23] = _LOAD_CONST_INLINE_BORROW, + [_RROT_3_r03] = _RROT_3, + [_RROT_3_r13] = _RROT_3, + [_RROT_3_r23] = _RROT_3, + [_RROT_3_r33] = _RROT_3, + [_START_EXECUTOR_r00] = _START_EXECUTOR, + [_MAKE_WARM_r00] = _MAKE_WARM, + [_MAKE_WARM_r11] = _MAKE_WARM, + [_MAKE_WARM_r22] = _MAKE_WARM, + [_MAKE_WARM_r33] = _MAKE_WARM, + [_FATAL_ERROR_r00] = _FATAL_ERROR, + [_FATAL_ERROR_r11] = _FATAL_ERROR, + [_FATAL_ERROR_r22] = _FATAL_ERROR, + [_FATAL_ERROR_r33] = _FATAL_ERROR, + [_DEOPT_r00] = _DEOPT, + [_DEOPT_r10] = _DEOPT, + [_DEOPT_r20] = _DEOPT, + [_DEOPT_r30] = _DEOPT, + [_HANDLE_PENDING_AND_DEOPT_r00] = _HANDLE_PENDING_AND_DEOPT, + [_HANDLE_PENDING_AND_DEOPT_r10] = _HANDLE_PENDING_AND_DEOPT, + [_HANDLE_PENDING_AND_DEOPT_r20] = _HANDLE_PENDING_AND_DEOPT, + [_HANDLE_PENDING_AND_DEOPT_r30] = _HANDLE_PENDING_AND_DEOPT, + [_ERROR_POP_N_r00] = _ERROR_POP_N, + [_SPILL_OR_RELOAD_r01] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r02] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r03] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r10] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r12] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r13] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r20] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r21] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r23] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r30] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r31] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r32] = _SPILL_OR_RELOAD, + [_TIER2_RESUME_CHECK_r00] = _TIER2_RESUME_CHECK, + [_TIER2_RESUME_CHECK_r11] = _TIER2_RESUME_CHECK, + [_TIER2_RESUME_CHECK_r22] = _TIER2_RESUME_CHECK, + [_TIER2_RESUME_CHECK_r33] = _TIER2_RESUME_CHECK, + [_COLD_EXIT_r00] = _COLD_EXIT, + [_COLD_DYNAMIC_EXIT_r00] = _COLD_DYNAMIC_EXIT, + [_GUARD_CODE_VERSION__PUSH_FRAME_r00] = _GUARD_CODE_VERSION__PUSH_FRAME, + [_GUARD_CODE_VERSION__PUSH_FRAME_r11] = _GUARD_CODE_VERSION__PUSH_FRAME, + [_GUARD_CODE_VERSION__PUSH_FRAME_r22] = _GUARD_CODE_VERSION__PUSH_FRAME, + [_GUARD_CODE_VERSION__PUSH_FRAME_r33] = _GUARD_CODE_VERSION__PUSH_FRAME, + [_GUARD_CODE_VERSION_YIELD_VALUE_r00] = _GUARD_CODE_VERSION_YIELD_VALUE, + [_GUARD_CODE_VERSION_YIELD_VALUE_r11] = _GUARD_CODE_VERSION_YIELD_VALUE, + [_GUARD_CODE_VERSION_YIELD_VALUE_r22] = _GUARD_CODE_VERSION_YIELD_VALUE, + [_GUARD_CODE_VERSION_YIELD_VALUE_r33] = _GUARD_CODE_VERSION_YIELD_VALUE, + [_GUARD_CODE_VERSION_RETURN_VALUE_r00] = _GUARD_CODE_VERSION_RETURN_VALUE, + [_GUARD_CODE_VERSION_RETURN_VALUE_r11] = _GUARD_CODE_VERSION_RETURN_VALUE, + [_GUARD_CODE_VERSION_RETURN_VALUE_r22] = _GUARD_CODE_VERSION_RETURN_VALUE, + [_GUARD_CODE_VERSION_RETURN_VALUE_r33] = _GUARD_CODE_VERSION_RETURN_VALUE, + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r00] = _GUARD_CODE_VERSION_RETURN_GENERATOR, + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r11] = _GUARD_CODE_VERSION_RETURN_GENERATOR, + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r22] = _GUARD_CODE_VERSION_RETURN_GENERATOR, + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r33] = _GUARD_CODE_VERSION_RETURN_GENERATOR, + [_GUARD_IP__PUSH_FRAME_r00] = _GUARD_IP__PUSH_FRAME, + [_GUARD_IP__PUSH_FRAME_r11] = _GUARD_IP__PUSH_FRAME, + [_GUARD_IP__PUSH_FRAME_r22] = _GUARD_IP__PUSH_FRAME, + [_GUARD_IP__PUSH_FRAME_r33] = _GUARD_IP__PUSH_FRAME, + [_GUARD_IP_YIELD_VALUE_r00] = _GUARD_IP_YIELD_VALUE, + [_GUARD_IP_YIELD_VALUE_r11] = _GUARD_IP_YIELD_VALUE, + [_GUARD_IP_YIELD_VALUE_r22] = _GUARD_IP_YIELD_VALUE, + [_GUARD_IP_YIELD_VALUE_r33] = _GUARD_IP_YIELD_VALUE, + [_GUARD_IP_RETURN_VALUE_r00] = _GUARD_IP_RETURN_VALUE, + [_GUARD_IP_RETURN_VALUE_r11] = _GUARD_IP_RETURN_VALUE, + [_GUARD_IP_RETURN_VALUE_r22] = _GUARD_IP_RETURN_VALUE, + [_GUARD_IP_RETURN_VALUE_r33] = _GUARD_IP_RETURN_VALUE, + [_GUARD_IP_RETURN_GENERATOR_r00] = _GUARD_IP_RETURN_GENERATOR, + [_GUARD_IP_RETURN_GENERATOR_r11] = _GUARD_IP_RETURN_GENERATOR, + [_GUARD_IP_RETURN_GENERATOR_r22] = _GUARD_IP_RETURN_GENERATOR, + [_GUARD_IP_RETURN_GENERATOR_r33] = _GUARD_IP_RETURN_GENERATOR, +}; + +const uint16_t _PyUop_SpillsAndReloads[4][4] = { + [0][1] = _SPILL_OR_RELOAD_r01, + [0][2] = _SPILL_OR_RELOAD_r02, + [0][3] = _SPILL_OR_RELOAD_r03, + [1][0] = _SPILL_OR_RELOAD_r10, + [1][2] = _SPILL_OR_RELOAD_r12, + [1][3] = _SPILL_OR_RELOAD_r13, + [2][0] = _SPILL_OR_RELOAD_r20, + [2][1] = _SPILL_OR_RELOAD_r21, + [2][3] = _SPILL_OR_RELOAD_r23, + [3][0] = _SPILL_OR_RELOAD_r30, + [3][1] = _SPILL_OR_RELOAD_r31, + [3][2] = _SPILL_OR_RELOAD_r32, +}; + +const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { + [_ALLOCATE_OBJECT] = "_ALLOCATE_OBJECT", + [_ALLOCATE_OBJECT_r00] = "_ALLOCATE_OBJECT_r00", [_BINARY_OP] = "_BINARY_OP", + [_BINARY_OP_r23] = "_BINARY_OP_r23", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", - [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS", + [_BINARY_OP_ADD_FLOAT_r03] = "_BINARY_OP_ADD_FLOAT_r03", + [_BINARY_OP_ADD_FLOAT_r13] = "_BINARY_OP_ADD_FLOAT_r13", + [_BINARY_OP_ADD_FLOAT_r23] = "_BINARY_OP_ADD_FLOAT_r23", + [_BINARY_OP_ADD_FLOAT_INPLACE] = "_BINARY_OP_ADD_FLOAT_INPLACE", + [_BINARY_OP_ADD_FLOAT_INPLACE_r03] = "_BINARY_OP_ADD_FLOAT_INPLACE_r03", + [_BINARY_OP_ADD_FLOAT_INPLACE_r13] = "_BINARY_OP_ADD_FLOAT_INPLACE_r13", + [_BINARY_OP_ADD_FLOAT_INPLACE_r23] = "_BINARY_OP_ADD_FLOAT_INPLACE_r23", + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT] = "_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT", + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03] = "_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03", + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13] = "_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13", + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23] = "_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", + [_BINARY_OP_ADD_INT_r03] = "_BINARY_OP_ADD_INT_r03", + [_BINARY_OP_ADD_INT_r13] = "_BINARY_OP_ADD_INT_r13", + [_BINARY_OP_ADD_INT_r23] = "_BINARY_OP_ADD_INT_r23", + [_BINARY_OP_ADD_INT_INPLACE] = "_BINARY_OP_ADD_INT_INPLACE", + [_BINARY_OP_ADD_INT_INPLACE_r03] = "_BINARY_OP_ADD_INT_INPLACE_r03", + [_BINARY_OP_ADD_INT_INPLACE_r13] = "_BINARY_OP_ADD_INT_INPLACE_r13", + [_BINARY_OP_ADD_INT_INPLACE_r23] = "_BINARY_OP_ADD_INT_INPLACE_r23", + [_BINARY_OP_ADD_INT_INPLACE_RIGHT] = "_BINARY_OP_ADD_INT_INPLACE_RIGHT", + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r03] = "_BINARY_OP_ADD_INT_INPLACE_RIGHT_r03", + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r13] = "_BINARY_OP_ADD_INT_INPLACE_RIGHT_r13", + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r23] = "_BINARY_OP_ADD_INT_INPLACE_RIGHT_r23", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", + [_BINARY_OP_ADD_UNICODE_r03] = "_BINARY_OP_ADD_UNICODE_r03", + [_BINARY_OP_ADD_UNICODE_r13] = "_BINARY_OP_ADD_UNICODE_r13", + [_BINARY_OP_ADD_UNICODE_r23] = "_BINARY_OP_ADD_UNICODE_r23", [_BINARY_OP_EXTEND] = "_BINARY_OP_EXTEND", + [_BINARY_OP_EXTEND_r23] = "_BINARY_OP_EXTEND_r23", [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", + [_BINARY_OP_INPLACE_ADD_UNICODE_r21] = "_BINARY_OP_INPLACE_ADD_UNICODE_r21", [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", - [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS", + [_BINARY_OP_MULTIPLY_FLOAT_r03] = "_BINARY_OP_MULTIPLY_FLOAT_r03", + [_BINARY_OP_MULTIPLY_FLOAT_r13] = "_BINARY_OP_MULTIPLY_FLOAT_r13", + [_BINARY_OP_MULTIPLY_FLOAT_r23] = "_BINARY_OP_MULTIPLY_FLOAT_r23", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23", [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", + [_BINARY_OP_MULTIPLY_INT_r03] = "_BINARY_OP_MULTIPLY_INT_r03", + [_BINARY_OP_MULTIPLY_INT_r13] = "_BINARY_OP_MULTIPLY_INT_r13", + [_BINARY_OP_MULTIPLY_INT_r23] = "_BINARY_OP_MULTIPLY_INT_r23", + [_BINARY_OP_MULTIPLY_INT_INPLACE] = "_BINARY_OP_MULTIPLY_INT_INPLACE", + [_BINARY_OP_MULTIPLY_INT_INPLACE_r03] = "_BINARY_OP_MULTIPLY_INT_INPLACE_r03", + [_BINARY_OP_MULTIPLY_INT_INPLACE_r13] = "_BINARY_OP_MULTIPLY_INT_INPLACE_r13", + [_BINARY_OP_MULTIPLY_INT_INPLACE_r23] = "_BINARY_OP_MULTIPLY_INT_INPLACE_r23", + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT] = "_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT", + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03] = "_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03", + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13] = "_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13", + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23] = "_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23", [_BINARY_OP_SUBSCR_CHECK_FUNC] = "_BINARY_OP_SUBSCR_CHECK_FUNC", + [_BINARY_OP_SUBSCR_CHECK_FUNC_r23] = "_BINARY_OP_SUBSCR_CHECK_FUNC_r23", [_BINARY_OP_SUBSCR_DICT] = "_BINARY_OP_SUBSCR_DICT", + [_BINARY_OP_SUBSCR_DICT_r23] = "_BINARY_OP_SUBSCR_DICT_r23", + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH] = "_BINARY_OP_SUBSCR_DICT_KNOWN_HASH", + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23] = "_BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23", [_BINARY_OP_SUBSCR_INIT_CALL] = "_BINARY_OP_SUBSCR_INIT_CALL", + [_BINARY_OP_SUBSCR_INIT_CALL_r01] = "_BINARY_OP_SUBSCR_INIT_CALL_r01", + [_BINARY_OP_SUBSCR_INIT_CALL_r11] = "_BINARY_OP_SUBSCR_INIT_CALL_r11", + [_BINARY_OP_SUBSCR_INIT_CALL_r21] = "_BINARY_OP_SUBSCR_INIT_CALL_r21", + [_BINARY_OP_SUBSCR_INIT_CALL_r31] = "_BINARY_OP_SUBSCR_INIT_CALL_r31", [_BINARY_OP_SUBSCR_LIST_INT] = "_BINARY_OP_SUBSCR_LIST_INT", + [_BINARY_OP_SUBSCR_LIST_INT_r23] = "_BINARY_OP_SUBSCR_LIST_INT_r23", [_BINARY_OP_SUBSCR_LIST_SLICE] = "_BINARY_OP_SUBSCR_LIST_SLICE", + [_BINARY_OP_SUBSCR_LIST_SLICE_r23] = "_BINARY_OP_SUBSCR_LIST_SLICE_r23", [_BINARY_OP_SUBSCR_STR_INT] = "_BINARY_OP_SUBSCR_STR_INT", + [_BINARY_OP_SUBSCR_STR_INT_r23] = "_BINARY_OP_SUBSCR_STR_INT_r23", [_BINARY_OP_SUBSCR_TUPLE_INT] = "_BINARY_OP_SUBSCR_TUPLE_INT", + [_BINARY_OP_SUBSCR_TUPLE_INT_r03] = "_BINARY_OP_SUBSCR_TUPLE_INT_r03", + [_BINARY_OP_SUBSCR_TUPLE_INT_r13] = "_BINARY_OP_SUBSCR_TUPLE_INT_r13", + [_BINARY_OP_SUBSCR_TUPLE_INT_r23] = "_BINARY_OP_SUBSCR_TUPLE_INT_r23", + [_BINARY_OP_SUBSCR_USTR_INT] = "_BINARY_OP_SUBSCR_USTR_INT", + [_BINARY_OP_SUBSCR_USTR_INT_r23] = "_BINARY_OP_SUBSCR_USTR_INT_r23", [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", - [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS", + [_BINARY_OP_SUBTRACT_FLOAT_r03] = "_BINARY_OP_SUBTRACT_FLOAT_r03", + [_BINARY_OP_SUBTRACT_FLOAT_r13] = "_BINARY_OP_SUBTRACT_FLOAT_r13", + [_BINARY_OP_SUBTRACT_FLOAT_r23] = "_BINARY_OP_SUBTRACT_FLOAT_r23", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23", [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", + [_BINARY_OP_SUBTRACT_INT_r03] = "_BINARY_OP_SUBTRACT_INT_r03", + [_BINARY_OP_SUBTRACT_INT_r13] = "_BINARY_OP_SUBTRACT_INT_r13", + [_BINARY_OP_SUBTRACT_INT_r23] = "_BINARY_OP_SUBTRACT_INT_r23", + [_BINARY_OP_SUBTRACT_INT_INPLACE] = "_BINARY_OP_SUBTRACT_INT_INPLACE", + [_BINARY_OP_SUBTRACT_INT_INPLACE_r03] = "_BINARY_OP_SUBTRACT_INT_INPLACE_r03", + [_BINARY_OP_SUBTRACT_INT_INPLACE_r13] = "_BINARY_OP_SUBTRACT_INT_INPLACE_r13", + [_BINARY_OP_SUBTRACT_INT_INPLACE_r23] = "_BINARY_OP_SUBTRACT_INT_INPLACE_r23", + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT] = "_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT", + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03] = "_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03", + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13] = "_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13", + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23] = "_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23", + [_BINARY_OP_TRUEDIV_FLOAT] = "_BINARY_OP_TRUEDIV_FLOAT", + [_BINARY_OP_TRUEDIV_FLOAT_r23] = "_BINARY_OP_TRUEDIV_FLOAT_r23", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23", [_BINARY_SLICE] = "_BINARY_SLICE", + [_BINARY_SLICE_r31] = "_BINARY_SLICE_r31", [_BUILD_INTERPOLATION] = "_BUILD_INTERPOLATION", + [_BUILD_INTERPOLATION_r01] = "_BUILD_INTERPOLATION_r01", [_BUILD_LIST] = "_BUILD_LIST", + [_BUILD_LIST_r01] = "_BUILD_LIST_r01", [_BUILD_MAP] = "_BUILD_MAP", + [_BUILD_MAP_r01] = "_BUILD_MAP_r01", [_BUILD_SET] = "_BUILD_SET", + [_BUILD_SET_r01] = "_BUILD_SET_r01", [_BUILD_SLICE] = "_BUILD_SLICE", + [_BUILD_SLICE_r01] = "_BUILD_SLICE_r01", [_BUILD_STRING] = "_BUILD_STRING", + [_BUILD_STRING_r01] = "_BUILD_STRING_r01", [_BUILD_TEMPLATE] = "_BUILD_TEMPLATE", + [_BUILD_TEMPLATE_r21] = "_BUILD_TEMPLATE_r21", [_BUILD_TUPLE] = "_BUILD_TUPLE", + [_BUILD_TUPLE_r01] = "_BUILD_TUPLE_r01", [_CALL_BUILTIN_CLASS] = "_CALL_BUILTIN_CLASS", + [_CALL_BUILTIN_CLASS_r00] = "_CALL_BUILTIN_CLASS_r00", [_CALL_BUILTIN_FAST] = "_CALL_BUILTIN_FAST", + [_CALL_BUILTIN_FAST_r00] = "_CALL_BUILTIN_FAST_r00", [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = "_CALL_BUILTIN_FAST_WITH_KEYWORDS", + [_CALL_BUILTIN_FAST_WITH_KEYWORDS_r00] = "_CALL_BUILTIN_FAST_WITH_KEYWORDS_r00", [_CALL_BUILTIN_O] = "_CALL_BUILTIN_O", + [_CALL_BUILTIN_O_r03] = "_CALL_BUILTIN_O_r03", + [_CALL_FUNCTION_EX_NON_PY_GENERAL] = "_CALL_FUNCTION_EX_NON_PY_GENERAL", + [_CALL_FUNCTION_EX_NON_PY_GENERAL_r31] = "_CALL_FUNCTION_EX_NON_PY_GENERAL_r31", [_CALL_INTRINSIC_1] = "_CALL_INTRINSIC_1", + [_CALL_INTRINSIC_1_r12] = "_CALL_INTRINSIC_1_r12", [_CALL_INTRINSIC_2] = "_CALL_INTRINSIC_2", + [_CALL_INTRINSIC_2_r23] = "_CALL_INTRINSIC_2_r23", [_CALL_ISINSTANCE] = "_CALL_ISINSTANCE", + [_CALL_ISINSTANCE_r31] = "_CALL_ISINSTANCE_r31", [_CALL_KW_NON_PY] = "_CALL_KW_NON_PY", + [_CALL_KW_NON_PY_r11] = "_CALL_KW_NON_PY_r11", [_CALL_LEN] = "_CALL_LEN", + [_CALL_LEN_r33] = "_CALL_LEN_r33", [_CALL_LIST_APPEND] = "_CALL_LIST_APPEND", + [_CALL_LIST_APPEND_r03] = "_CALL_LIST_APPEND_r03", + [_CALL_LIST_APPEND_r13] = "_CALL_LIST_APPEND_r13", + [_CALL_LIST_APPEND_r23] = "_CALL_LIST_APPEND_r23", + [_CALL_LIST_APPEND_r33] = "_CALL_LIST_APPEND_r33", [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", + [_CALL_METHOD_DESCRIPTOR_FAST_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_r00", + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = "_CALL_METHOD_DESCRIPTOR_FAST_INLINE", + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00", [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00", + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE", + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00", [_CALL_METHOD_DESCRIPTOR_NOARGS] = "_CALL_METHOD_DESCRIPTOR_NOARGS", + [_CALL_METHOD_DESCRIPTOR_NOARGS_r03] = "_CALL_METHOD_DESCRIPTOR_NOARGS_r03", + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE] = "_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE", + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03] = "_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03", [_CALL_METHOD_DESCRIPTOR_O] = "_CALL_METHOD_DESCRIPTOR_O", + [_CALL_METHOD_DESCRIPTOR_O_r03] = "_CALL_METHOD_DESCRIPTOR_O_r03", + [_CALL_METHOD_DESCRIPTOR_O_INLINE] = "_CALL_METHOD_DESCRIPTOR_O_INLINE", + [_CALL_METHOD_DESCRIPTOR_O_INLINE_r03] = "_CALL_METHOD_DESCRIPTOR_O_INLINE_r03", [_CALL_NON_PY_GENERAL] = "_CALL_NON_PY_GENERAL", + [_CALL_NON_PY_GENERAL_r01] = "_CALL_NON_PY_GENERAL_r01", [_CALL_STR_1] = "_CALL_STR_1", + [_CALL_STR_1_r32] = "_CALL_STR_1_r32", [_CALL_TUPLE_1] = "_CALL_TUPLE_1", + [_CALL_TUPLE_1_r32] = "_CALL_TUPLE_1_r32", [_CALL_TYPE_1] = "_CALL_TYPE_1", - [_CHECK_AND_ALLOCATE_OBJECT] = "_CHECK_AND_ALLOCATE_OBJECT", + [_CALL_TYPE_1_r02] = "_CALL_TYPE_1_r02", + [_CALL_TYPE_1_r12] = "_CALL_TYPE_1_r12", + [_CALL_TYPE_1_r22] = "_CALL_TYPE_1_r22", + [_CALL_TYPE_1_r32] = "_CALL_TYPE_1_r32", [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", + [_CHECK_ATTR_CLASS_r01] = "_CHECK_ATTR_CLASS_r01", + [_CHECK_ATTR_CLASS_r11] = "_CHECK_ATTR_CLASS_r11", + [_CHECK_ATTR_CLASS_r22] = "_CHECK_ATTR_CLASS_r22", + [_CHECK_ATTR_CLASS_r33] = "_CHECK_ATTR_CLASS_r33", [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", + [_CHECK_ATTR_METHOD_LAZY_DICT_r01] = "_CHECK_ATTR_METHOD_LAZY_DICT_r01", + [_CHECK_ATTR_METHOD_LAZY_DICT_r11] = "_CHECK_ATTR_METHOD_LAZY_DICT_r11", + [_CHECK_ATTR_METHOD_LAZY_DICT_r22] = "_CHECK_ATTR_METHOD_LAZY_DICT_r22", + [_CHECK_ATTR_METHOD_LAZY_DICT_r33] = "_CHECK_ATTR_METHOD_LAZY_DICT_r33", [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00", [_CHECK_EG_MATCH] = "_CHECK_EG_MATCH", + [_CHECK_EG_MATCH_r22] = "_CHECK_EG_MATCH_r22", [_CHECK_EXC_MATCH] = "_CHECK_EXC_MATCH", - [_CHECK_FUNCTION] = "_CHECK_FUNCTION", + [_CHECK_EXC_MATCH_r22] = "_CHECK_EXC_MATCH_r22", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", + [_CHECK_FUNCTION_EXACT_ARGS_r00] = "_CHECK_FUNCTION_EXACT_ARGS_r00", [_CHECK_FUNCTION_VERSION] = "_CHECK_FUNCTION_VERSION", + [_CHECK_FUNCTION_VERSION_r00] = "_CHECK_FUNCTION_VERSION_r00", [_CHECK_FUNCTION_VERSION_INLINE] = "_CHECK_FUNCTION_VERSION_INLINE", + [_CHECK_FUNCTION_VERSION_INLINE_r00] = "_CHECK_FUNCTION_VERSION_INLINE_r00", + [_CHECK_FUNCTION_VERSION_INLINE_r11] = "_CHECK_FUNCTION_VERSION_INLINE_r11", + [_CHECK_FUNCTION_VERSION_INLINE_r22] = "_CHECK_FUNCTION_VERSION_INLINE_r22", + [_CHECK_FUNCTION_VERSION_INLINE_r33] = "_CHECK_FUNCTION_VERSION_INLINE_r33", [_CHECK_FUNCTION_VERSION_KW] = "_CHECK_FUNCTION_VERSION_KW", + [_CHECK_FUNCTION_VERSION_KW_r11] = "_CHECK_FUNCTION_VERSION_KW_r11", [_CHECK_IS_NOT_PY_CALLABLE] = "_CHECK_IS_NOT_PY_CALLABLE", + [_CHECK_IS_NOT_PY_CALLABLE_r00] = "_CHECK_IS_NOT_PY_CALLABLE_r00", + [_CHECK_IS_NOT_PY_CALLABLE_EX] = "_CHECK_IS_NOT_PY_CALLABLE_EX", + [_CHECK_IS_NOT_PY_CALLABLE_EX_r03] = "_CHECK_IS_NOT_PY_CALLABLE_EX_r03", + [_CHECK_IS_NOT_PY_CALLABLE_EX_r13] = "_CHECK_IS_NOT_PY_CALLABLE_EX_r13", + [_CHECK_IS_NOT_PY_CALLABLE_EX_r23] = "_CHECK_IS_NOT_PY_CALLABLE_EX_r23", + [_CHECK_IS_NOT_PY_CALLABLE_EX_r33] = "_CHECK_IS_NOT_PY_CALLABLE_EX_r33", [_CHECK_IS_NOT_PY_CALLABLE_KW] = "_CHECK_IS_NOT_PY_CALLABLE_KW", + [_CHECK_IS_NOT_PY_CALLABLE_KW_r11] = "_CHECK_IS_NOT_PY_CALLABLE_KW_r11", + [_CHECK_IS_PY_CALLABLE_EX] = "_CHECK_IS_PY_CALLABLE_EX", + [_CHECK_IS_PY_CALLABLE_EX_r03] = "_CHECK_IS_PY_CALLABLE_EX_r03", + [_CHECK_IS_PY_CALLABLE_EX_r13] = "_CHECK_IS_PY_CALLABLE_EX_r13", + [_CHECK_IS_PY_CALLABLE_EX_r23] = "_CHECK_IS_PY_CALLABLE_EX_r23", + [_CHECK_IS_PY_CALLABLE_EX_r33] = "_CHECK_IS_PY_CALLABLE_EX_r33", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r01] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r01", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r11] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r11", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r22] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r22", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r33] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r33", [_CHECK_METHOD_VERSION] = "_CHECK_METHOD_VERSION", + [_CHECK_METHOD_VERSION_r00] = "_CHECK_METHOD_VERSION_r00", [_CHECK_METHOD_VERSION_KW] = "_CHECK_METHOD_VERSION_KW", + [_CHECK_METHOD_VERSION_KW_r11] = "_CHECK_METHOD_VERSION_KW_r11", + [_CHECK_OBJECT] = "_CHECK_OBJECT", + [_CHECK_OBJECT_r00] = "_CHECK_OBJECT_r00", [_CHECK_PEP_523] = "_CHECK_PEP_523", + [_CHECK_PEP_523_r00] = "_CHECK_PEP_523_r00", + [_CHECK_PEP_523_r11] = "_CHECK_PEP_523_r11", + [_CHECK_PEP_523_r22] = "_CHECK_PEP_523_r22", + [_CHECK_PEP_523_r33] = "_CHECK_PEP_523_r33", [_CHECK_PERIODIC] = "_CHECK_PERIODIC", + [_CHECK_PERIODIC_r00] = "_CHECK_PERIODIC_r00", [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = "_CHECK_PERIODIC_IF_NOT_YIELD_FROM", + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00] = "_CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00", + [_CHECK_RECURSION_LIMIT] = "_CHECK_RECURSION_LIMIT", + [_CHECK_RECURSION_LIMIT_r00] = "_CHECK_RECURSION_LIMIT_r00", + [_CHECK_RECURSION_LIMIT_r11] = "_CHECK_RECURSION_LIMIT_r11", + [_CHECK_RECURSION_LIMIT_r22] = "_CHECK_RECURSION_LIMIT_r22", + [_CHECK_RECURSION_LIMIT_r33] = "_CHECK_RECURSION_LIMIT_r33", [_CHECK_RECURSION_REMAINING] = "_CHECK_RECURSION_REMAINING", + [_CHECK_RECURSION_REMAINING_r00] = "_CHECK_RECURSION_REMAINING_r00", + [_CHECK_RECURSION_REMAINING_r11] = "_CHECK_RECURSION_REMAINING_r11", + [_CHECK_RECURSION_REMAINING_r22] = "_CHECK_RECURSION_REMAINING_r22", + [_CHECK_RECURSION_REMAINING_r33] = "_CHECK_RECURSION_REMAINING_r33", [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", + [_CHECK_STACK_SPACE_r00] = "_CHECK_STACK_SPACE_r00", [_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND", + [_CHECK_STACK_SPACE_OPERAND_r00] = "_CHECK_STACK_SPACE_OPERAND_r00", + [_CHECK_STACK_SPACE_OPERAND_r11] = "_CHECK_STACK_SPACE_OPERAND_r11", + [_CHECK_STACK_SPACE_OPERAND_r22] = "_CHECK_STACK_SPACE_OPERAND_r22", + [_CHECK_STACK_SPACE_OPERAND_r33] = "_CHECK_STACK_SPACE_OPERAND_r33", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", + [_CHECK_VALIDITY_r00] = "_CHECK_VALIDITY_r00", + [_CHECK_VALIDITY_r11] = "_CHECK_VALIDITY_r11", + [_CHECK_VALIDITY_r22] = "_CHECK_VALIDITY_r22", + [_CHECK_VALIDITY_r33] = "_CHECK_VALIDITY_r33", + [_COLD_DYNAMIC_EXIT] = "_COLD_DYNAMIC_EXIT", + [_COLD_DYNAMIC_EXIT_r00] = "_COLD_DYNAMIC_EXIT_r00", [_COLD_EXIT] = "_COLD_EXIT", + [_COLD_EXIT_r00] = "_COLD_EXIT_r00", [_COMPARE_OP] = "_COMPARE_OP", + [_COMPARE_OP_r21] = "_COMPARE_OP_r21", [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", + [_COMPARE_OP_FLOAT_r03] = "_COMPARE_OP_FLOAT_r03", + [_COMPARE_OP_FLOAT_r13] = "_COMPARE_OP_FLOAT_r13", + [_COMPARE_OP_FLOAT_r23] = "_COMPARE_OP_FLOAT_r23", [_COMPARE_OP_INT] = "_COMPARE_OP_INT", + [_COMPARE_OP_INT_r23] = "_COMPARE_OP_INT_r23", [_COMPARE_OP_STR] = "_COMPARE_OP_STR", + [_COMPARE_OP_STR_r23] = "_COMPARE_OP_STR_r23", [_CONTAINS_OP] = "_CONTAINS_OP", + [_CONTAINS_OP_r23] = "_CONTAINS_OP_r23", [_CONTAINS_OP_DICT] = "_CONTAINS_OP_DICT", + [_CONTAINS_OP_DICT_r23] = "_CONTAINS_OP_DICT_r23", [_CONTAINS_OP_SET] = "_CONTAINS_OP_SET", + [_CONTAINS_OP_SET_r23] = "_CONTAINS_OP_SET_r23", [_CONVERT_VALUE] = "_CONVERT_VALUE", + [_CONVERT_VALUE_r11] = "_CONVERT_VALUE_r11", [_COPY] = "_COPY", + [_COPY_r01] = "_COPY_r01", [_COPY_1] = "_COPY_1", + [_COPY_1_r02] = "_COPY_1_r02", + [_COPY_1_r12] = "_COPY_1_r12", + [_COPY_1_r23] = "_COPY_1_r23", [_COPY_2] = "_COPY_2", + [_COPY_2_r03] = "_COPY_2_r03", + [_COPY_2_r13] = "_COPY_2_r13", + [_COPY_2_r23] = "_COPY_2_r23", [_COPY_3] = "_COPY_3", + [_COPY_3_r03] = "_COPY_3_r03", + [_COPY_3_r13] = "_COPY_3_r13", + [_COPY_3_r23] = "_COPY_3_r23", + [_COPY_3_r33] = "_COPY_3_r33", [_COPY_FREE_VARS] = "_COPY_FREE_VARS", + [_COPY_FREE_VARS_r00] = "_COPY_FREE_VARS_r00", + [_COPY_FREE_VARS_r11] = "_COPY_FREE_VARS_r11", + [_COPY_FREE_VARS_r22] = "_COPY_FREE_VARS_r22", + [_COPY_FREE_VARS_r33] = "_COPY_FREE_VARS_r33", [_CREATE_INIT_FRAME] = "_CREATE_INIT_FRAME", + [_CREATE_INIT_FRAME_r01] = "_CREATE_INIT_FRAME_r01", [_DELETE_ATTR] = "_DELETE_ATTR", + [_DELETE_ATTR_r10] = "_DELETE_ATTR_r10", [_DELETE_DEREF] = "_DELETE_DEREF", + [_DELETE_DEREF_r00] = "_DELETE_DEREF_r00", [_DELETE_FAST] = "_DELETE_FAST", + [_DELETE_FAST_r00] = "_DELETE_FAST_r00", [_DELETE_GLOBAL] = "_DELETE_GLOBAL", + [_DELETE_GLOBAL_r00] = "_DELETE_GLOBAL_r00", [_DELETE_NAME] = "_DELETE_NAME", + [_DELETE_NAME_r00] = "_DELETE_NAME_r00", [_DELETE_SUBSCR] = "_DELETE_SUBSCR", + [_DELETE_SUBSCR_r20] = "_DELETE_SUBSCR_r20", [_DEOPT] = "_DEOPT", + [_DEOPT_r00] = "_DEOPT_r00", + [_DEOPT_r10] = "_DEOPT_r10", + [_DEOPT_r20] = "_DEOPT_r20", + [_DEOPT_r30] = "_DEOPT_r30", [_DICT_MERGE] = "_DICT_MERGE", + [_DICT_MERGE_r11] = "_DICT_MERGE_r11", [_DICT_UPDATE] = "_DICT_UPDATE", + [_DICT_UPDATE_r11] = "_DICT_UPDATE_r11", + [_DYNAMIC_EXIT] = "_DYNAMIC_EXIT", + [_DYNAMIC_EXIT_r00] = "_DYNAMIC_EXIT_r00", + [_DYNAMIC_EXIT_r10] = "_DYNAMIC_EXIT_r10", + [_DYNAMIC_EXIT_r20] = "_DYNAMIC_EXIT_r20", + [_DYNAMIC_EXIT_r30] = "_DYNAMIC_EXIT_r30", [_END_FOR] = "_END_FOR", + [_END_FOR_r10] = "_END_FOR_r10", [_END_SEND] = "_END_SEND", + [_END_SEND_r31] = "_END_SEND_r31", [_ERROR_POP_N] = "_ERROR_POP_N", + [_ERROR_POP_N_r00] = "_ERROR_POP_N_r00", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", + [_EXIT_INIT_CHECK_r10] = "_EXIT_INIT_CHECK_r10", [_EXIT_TRACE] = "_EXIT_TRACE", + [_EXIT_TRACE_r00] = "_EXIT_TRACE_r00", + [_EXIT_TRACE_r10] = "_EXIT_TRACE_r10", + [_EXIT_TRACE_r20] = "_EXIT_TRACE_r20", + [_EXIT_TRACE_r30] = "_EXIT_TRACE_r30", [_EXPAND_METHOD] = "_EXPAND_METHOD", + [_EXPAND_METHOD_r00] = "_EXPAND_METHOD_r00", [_EXPAND_METHOD_KW] = "_EXPAND_METHOD_KW", + [_EXPAND_METHOD_KW_r11] = "_EXPAND_METHOD_KW_r11", [_FATAL_ERROR] = "_FATAL_ERROR", + [_FATAL_ERROR_r00] = "_FATAL_ERROR_r00", + [_FATAL_ERROR_r11] = "_FATAL_ERROR_r11", + [_FATAL_ERROR_r22] = "_FATAL_ERROR_r22", + [_FATAL_ERROR_r33] = "_FATAL_ERROR_r33", [_FORMAT_SIMPLE] = "_FORMAT_SIMPLE", + [_FORMAT_SIMPLE_r11] = "_FORMAT_SIMPLE_r11", [_FORMAT_WITH_SPEC] = "_FORMAT_WITH_SPEC", + [_FORMAT_WITH_SPEC_r21] = "_FORMAT_WITH_SPEC_r21", [_FOR_ITER_GEN_FRAME] = "_FOR_ITER_GEN_FRAME", + [_FOR_ITER_GEN_FRAME_r03] = "_FOR_ITER_GEN_FRAME_r03", + [_FOR_ITER_GEN_FRAME_r13] = "_FOR_ITER_GEN_FRAME_r13", + [_FOR_ITER_GEN_FRAME_r23] = "_FOR_ITER_GEN_FRAME_r23", [_FOR_ITER_TIER_TWO] = "_FOR_ITER_TIER_TWO", + [_FOR_ITER_TIER_TWO_r23] = "_FOR_ITER_TIER_TWO_r23", + [_FOR_ITER_VIRTUAL_TIER_TWO] = "_FOR_ITER_VIRTUAL_TIER_TWO", + [_FOR_ITER_VIRTUAL_TIER_TWO_r23] = "_FOR_ITER_VIRTUAL_TIER_TWO_r23", [_GET_AITER] = "_GET_AITER", + [_GET_AITER_r11] = "_GET_AITER_r11", [_GET_ANEXT] = "_GET_ANEXT", + [_GET_ANEXT_r12] = "_GET_ANEXT_r12", [_GET_AWAITABLE] = "_GET_AWAITABLE", + [_GET_AWAITABLE_r11] = "_GET_AWAITABLE_r11", [_GET_ITER] = "_GET_ITER", + [_GET_ITER_r12] = "_GET_ITER_r12", + [_GET_ITER_TRAD] = "_GET_ITER_TRAD", + [_GET_ITER_TRAD_r12] = "_GET_ITER_TRAD_r12", [_GET_LEN] = "_GET_LEN", - [_GET_YIELD_FROM_ITER] = "_GET_YIELD_FROM_ITER", + [_GET_LEN_r12] = "_GET_LEN_r12", [_GUARD_BINARY_OP_EXTEND] = "_GUARD_BINARY_OP_EXTEND", + [_GUARD_BINARY_OP_EXTEND_r22] = "_GUARD_BINARY_OP_EXTEND_r22", + [_GUARD_BINARY_OP_EXTEND_LHS] = "_GUARD_BINARY_OP_EXTEND_LHS", + [_GUARD_BINARY_OP_EXTEND_LHS_r02] = "_GUARD_BINARY_OP_EXTEND_LHS_r02", + [_GUARD_BINARY_OP_EXTEND_LHS_r12] = "_GUARD_BINARY_OP_EXTEND_LHS_r12", + [_GUARD_BINARY_OP_EXTEND_LHS_r22] = "_GUARD_BINARY_OP_EXTEND_LHS_r22", + [_GUARD_BINARY_OP_EXTEND_LHS_r33] = "_GUARD_BINARY_OP_EXTEND_LHS_r33", + [_GUARD_BINARY_OP_EXTEND_RHS] = "_GUARD_BINARY_OP_EXTEND_RHS", + [_GUARD_BINARY_OP_EXTEND_RHS_r02] = "_GUARD_BINARY_OP_EXTEND_RHS_r02", + [_GUARD_BINARY_OP_EXTEND_RHS_r12] = "_GUARD_BINARY_OP_EXTEND_RHS_r12", + [_GUARD_BINARY_OP_EXTEND_RHS_r22] = "_GUARD_BINARY_OP_EXTEND_RHS_r22", + [_GUARD_BINARY_OP_EXTEND_RHS_r33] = "_GUARD_BINARY_OP_EXTEND_RHS_r33", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33", + [_GUARD_BIT_IS_SET_POP] = "_GUARD_BIT_IS_SET_POP", + [_GUARD_BIT_IS_SET_POP_r00] = "_GUARD_BIT_IS_SET_POP_r00", + [_GUARD_BIT_IS_SET_POP_r10] = "_GUARD_BIT_IS_SET_POP_r10", + [_GUARD_BIT_IS_SET_POP_r21] = "_GUARD_BIT_IS_SET_POP_r21", + [_GUARD_BIT_IS_SET_POP_r32] = "_GUARD_BIT_IS_SET_POP_r32", + [_GUARD_BIT_IS_SET_POP_4] = "_GUARD_BIT_IS_SET_POP_4", + [_GUARD_BIT_IS_SET_POP_4_r00] = "_GUARD_BIT_IS_SET_POP_4_r00", + [_GUARD_BIT_IS_SET_POP_4_r10] = "_GUARD_BIT_IS_SET_POP_4_r10", + [_GUARD_BIT_IS_SET_POP_4_r21] = "_GUARD_BIT_IS_SET_POP_4_r21", + [_GUARD_BIT_IS_SET_POP_4_r32] = "_GUARD_BIT_IS_SET_POP_4_r32", + [_GUARD_BIT_IS_SET_POP_5] = "_GUARD_BIT_IS_SET_POP_5", + [_GUARD_BIT_IS_SET_POP_5_r00] = "_GUARD_BIT_IS_SET_POP_5_r00", + [_GUARD_BIT_IS_SET_POP_5_r10] = "_GUARD_BIT_IS_SET_POP_5_r10", + [_GUARD_BIT_IS_SET_POP_5_r21] = "_GUARD_BIT_IS_SET_POP_5_r21", + [_GUARD_BIT_IS_SET_POP_5_r32] = "_GUARD_BIT_IS_SET_POP_5_r32", + [_GUARD_BIT_IS_SET_POP_6] = "_GUARD_BIT_IS_SET_POP_6", + [_GUARD_BIT_IS_SET_POP_6_r00] = "_GUARD_BIT_IS_SET_POP_6_r00", + [_GUARD_BIT_IS_SET_POP_6_r10] = "_GUARD_BIT_IS_SET_POP_6_r10", + [_GUARD_BIT_IS_SET_POP_6_r21] = "_GUARD_BIT_IS_SET_POP_6_r21", + [_GUARD_BIT_IS_SET_POP_6_r32] = "_GUARD_BIT_IS_SET_POP_6_r32", + [_GUARD_BIT_IS_SET_POP_7] = "_GUARD_BIT_IS_SET_POP_7", + [_GUARD_BIT_IS_SET_POP_7_r00] = "_GUARD_BIT_IS_SET_POP_7_r00", + [_GUARD_BIT_IS_SET_POP_7_r10] = "_GUARD_BIT_IS_SET_POP_7_r10", + [_GUARD_BIT_IS_SET_POP_7_r21] = "_GUARD_BIT_IS_SET_POP_7_r21", + [_GUARD_BIT_IS_SET_POP_7_r32] = "_GUARD_BIT_IS_SET_POP_7_r32", + [_GUARD_BIT_IS_UNSET_POP] = "_GUARD_BIT_IS_UNSET_POP", + [_GUARD_BIT_IS_UNSET_POP_r00] = "_GUARD_BIT_IS_UNSET_POP_r00", + [_GUARD_BIT_IS_UNSET_POP_r10] = "_GUARD_BIT_IS_UNSET_POP_r10", + [_GUARD_BIT_IS_UNSET_POP_r21] = "_GUARD_BIT_IS_UNSET_POP_r21", + [_GUARD_BIT_IS_UNSET_POP_r32] = "_GUARD_BIT_IS_UNSET_POP_r32", + [_GUARD_BIT_IS_UNSET_POP_4] = "_GUARD_BIT_IS_UNSET_POP_4", + [_GUARD_BIT_IS_UNSET_POP_4_r00] = "_GUARD_BIT_IS_UNSET_POP_4_r00", + [_GUARD_BIT_IS_UNSET_POP_4_r10] = "_GUARD_BIT_IS_UNSET_POP_4_r10", + [_GUARD_BIT_IS_UNSET_POP_4_r21] = "_GUARD_BIT_IS_UNSET_POP_4_r21", + [_GUARD_BIT_IS_UNSET_POP_4_r32] = "_GUARD_BIT_IS_UNSET_POP_4_r32", + [_GUARD_BIT_IS_UNSET_POP_5] = "_GUARD_BIT_IS_UNSET_POP_5", + [_GUARD_BIT_IS_UNSET_POP_5_r00] = "_GUARD_BIT_IS_UNSET_POP_5_r00", + [_GUARD_BIT_IS_UNSET_POP_5_r10] = "_GUARD_BIT_IS_UNSET_POP_5_r10", + [_GUARD_BIT_IS_UNSET_POP_5_r21] = "_GUARD_BIT_IS_UNSET_POP_5_r21", + [_GUARD_BIT_IS_UNSET_POP_5_r32] = "_GUARD_BIT_IS_UNSET_POP_5_r32", + [_GUARD_BIT_IS_UNSET_POP_6] = "_GUARD_BIT_IS_UNSET_POP_6", + [_GUARD_BIT_IS_UNSET_POP_6_r00] = "_GUARD_BIT_IS_UNSET_POP_6_r00", + [_GUARD_BIT_IS_UNSET_POP_6_r10] = "_GUARD_BIT_IS_UNSET_POP_6_r10", + [_GUARD_BIT_IS_UNSET_POP_6_r21] = "_GUARD_BIT_IS_UNSET_POP_6_r21", + [_GUARD_BIT_IS_UNSET_POP_6_r32] = "_GUARD_BIT_IS_UNSET_POP_6_r32", + [_GUARD_BIT_IS_UNSET_POP_7] = "_GUARD_BIT_IS_UNSET_POP_7", + [_GUARD_BIT_IS_UNSET_POP_7_r00] = "_GUARD_BIT_IS_UNSET_POP_7_r00", + [_GUARD_BIT_IS_UNSET_POP_7_r10] = "_GUARD_BIT_IS_UNSET_POP_7_r10", + [_GUARD_BIT_IS_UNSET_POP_7_r21] = "_GUARD_BIT_IS_UNSET_POP_7_r21", + [_GUARD_BIT_IS_UNSET_POP_7_r32] = "_GUARD_BIT_IS_UNSET_POP_7_r32", + [_GUARD_CALLABLE_BUILTIN_CLASS] = "_GUARD_CALLABLE_BUILTIN_CLASS", + [_GUARD_CALLABLE_BUILTIN_CLASS_r00] = "_GUARD_CALLABLE_BUILTIN_CLASS_r00", + [_GUARD_CALLABLE_BUILTIN_FAST] = "_GUARD_CALLABLE_BUILTIN_FAST", + [_GUARD_CALLABLE_BUILTIN_FAST_r00] = "_GUARD_CALLABLE_BUILTIN_FAST_r00", + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS] = "_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS", + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00] = "_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00", + [_GUARD_CALLABLE_BUILTIN_O] = "_GUARD_CALLABLE_BUILTIN_O", + [_GUARD_CALLABLE_BUILTIN_O_r00] = "_GUARD_CALLABLE_BUILTIN_O_r00", [_GUARD_CALLABLE_ISINSTANCE] = "_GUARD_CALLABLE_ISINSTANCE", + [_GUARD_CALLABLE_ISINSTANCE_r03] = "_GUARD_CALLABLE_ISINSTANCE_r03", + [_GUARD_CALLABLE_ISINSTANCE_r13] = "_GUARD_CALLABLE_ISINSTANCE_r13", + [_GUARD_CALLABLE_ISINSTANCE_r23] = "_GUARD_CALLABLE_ISINSTANCE_r23", + [_GUARD_CALLABLE_ISINSTANCE_r33] = "_GUARD_CALLABLE_ISINSTANCE_r33", [_GUARD_CALLABLE_LEN] = "_GUARD_CALLABLE_LEN", + [_GUARD_CALLABLE_LEN_r03] = "_GUARD_CALLABLE_LEN_r03", + [_GUARD_CALLABLE_LEN_r13] = "_GUARD_CALLABLE_LEN_r13", + [_GUARD_CALLABLE_LEN_r23] = "_GUARD_CALLABLE_LEN_r23", + [_GUARD_CALLABLE_LEN_r33] = "_GUARD_CALLABLE_LEN_r33", [_GUARD_CALLABLE_LIST_APPEND] = "_GUARD_CALLABLE_LIST_APPEND", + [_GUARD_CALLABLE_LIST_APPEND_r03] = "_GUARD_CALLABLE_LIST_APPEND_r03", + [_GUARD_CALLABLE_LIST_APPEND_r13] = "_GUARD_CALLABLE_LIST_APPEND_r13", + [_GUARD_CALLABLE_LIST_APPEND_r23] = "_GUARD_CALLABLE_LIST_APPEND_r23", + [_GUARD_CALLABLE_LIST_APPEND_r33] = "_GUARD_CALLABLE_LIST_APPEND_r33", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_O", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00", [_GUARD_CALLABLE_STR_1] = "_GUARD_CALLABLE_STR_1", + [_GUARD_CALLABLE_STR_1_r03] = "_GUARD_CALLABLE_STR_1_r03", + [_GUARD_CALLABLE_STR_1_r13] = "_GUARD_CALLABLE_STR_1_r13", + [_GUARD_CALLABLE_STR_1_r23] = "_GUARD_CALLABLE_STR_1_r23", + [_GUARD_CALLABLE_STR_1_r33] = "_GUARD_CALLABLE_STR_1_r33", [_GUARD_CALLABLE_TUPLE_1] = "_GUARD_CALLABLE_TUPLE_1", + [_GUARD_CALLABLE_TUPLE_1_r03] = "_GUARD_CALLABLE_TUPLE_1_r03", + [_GUARD_CALLABLE_TUPLE_1_r13] = "_GUARD_CALLABLE_TUPLE_1_r13", + [_GUARD_CALLABLE_TUPLE_1_r23] = "_GUARD_CALLABLE_TUPLE_1_r23", + [_GUARD_CALLABLE_TUPLE_1_r33] = "_GUARD_CALLABLE_TUPLE_1_r33", [_GUARD_CALLABLE_TYPE_1] = "_GUARD_CALLABLE_TYPE_1", + [_GUARD_CALLABLE_TYPE_1_r03] = "_GUARD_CALLABLE_TYPE_1_r03", + [_GUARD_CALLABLE_TYPE_1_r13] = "_GUARD_CALLABLE_TYPE_1_r13", + [_GUARD_CALLABLE_TYPE_1_r23] = "_GUARD_CALLABLE_TYPE_1_r23", + [_GUARD_CALLABLE_TYPE_1_r33] = "_GUARD_CALLABLE_TYPE_1_r33", + [_GUARD_CODE_VERSION_RETURN_GENERATOR] = "_GUARD_CODE_VERSION_RETURN_GENERATOR", + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r00] = "_GUARD_CODE_VERSION_RETURN_GENERATOR_r00", + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r11] = "_GUARD_CODE_VERSION_RETURN_GENERATOR_r11", + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r22] = "_GUARD_CODE_VERSION_RETURN_GENERATOR_r22", + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r33] = "_GUARD_CODE_VERSION_RETURN_GENERATOR_r33", + [_GUARD_CODE_VERSION_RETURN_VALUE] = "_GUARD_CODE_VERSION_RETURN_VALUE", + [_GUARD_CODE_VERSION_RETURN_VALUE_r00] = "_GUARD_CODE_VERSION_RETURN_VALUE_r00", + [_GUARD_CODE_VERSION_RETURN_VALUE_r11] = "_GUARD_CODE_VERSION_RETURN_VALUE_r11", + [_GUARD_CODE_VERSION_RETURN_VALUE_r22] = "_GUARD_CODE_VERSION_RETURN_VALUE_r22", + [_GUARD_CODE_VERSION_RETURN_VALUE_r33] = "_GUARD_CODE_VERSION_RETURN_VALUE_r33", + [_GUARD_CODE_VERSION_YIELD_VALUE] = "_GUARD_CODE_VERSION_YIELD_VALUE", + [_GUARD_CODE_VERSION_YIELD_VALUE_r00] = "_GUARD_CODE_VERSION_YIELD_VALUE_r00", + [_GUARD_CODE_VERSION_YIELD_VALUE_r11] = "_GUARD_CODE_VERSION_YIELD_VALUE_r11", + [_GUARD_CODE_VERSION_YIELD_VALUE_r22] = "_GUARD_CODE_VERSION_YIELD_VALUE_r22", + [_GUARD_CODE_VERSION_YIELD_VALUE_r33] = "_GUARD_CODE_VERSION_YIELD_VALUE_r33", + [_GUARD_CODE_VERSION__PUSH_FRAME] = "_GUARD_CODE_VERSION__PUSH_FRAME", + [_GUARD_CODE_VERSION__PUSH_FRAME_r00] = "_GUARD_CODE_VERSION__PUSH_FRAME_r00", + [_GUARD_CODE_VERSION__PUSH_FRAME_r11] = "_GUARD_CODE_VERSION__PUSH_FRAME_r11", + [_GUARD_CODE_VERSION__PUSH_FRAME_r22] = "_GUARD_CODE_VERSION__PUSH_FRAME_r22", + [_GUARD_CODE_VERSION__PUSH_FRAME_r33] = "_GUARD_CODE_VERSION__PUSH_FRAME_r33", [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", + [_GUARD_DORV_NO_DICT_r01] = "_GUARD_DORV_NO_DICT_r01", + [_GUARD_DORV_NO_DICT_r11] = "_GUARD_DORV_NO_DICT_r11", + [_GUARD_DORV_NO_DICT_r22] = "_GUARD_DORV_NO_DICT_r22", + [_GUARD_DORV_NO_DICT_r33] = "_GUARD_DORV_NO_DICT_r33", [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", + [_GUARD_GLOBALS_VERSION_r00] = "_GUARD_GLOBALS_VERSION_r00", + [_GUARD_GLOBALS_VERSION_r11] = "_GUARD_GLOBALS_VERSION_r11", + [_GUARD_GLOBALS_VERSION_r22] = "_GUARD_GLOBALS_VERSION_r22", + [_GUARD_GLOBALS_VERSION_r33] = "_GUARD_GLOBALS_VERSION_r33", + [_GUARD_IP_RETURN_GENERATOR] = "_GUARD_IP_RETURN_GENERATOR", + [_GUARD_IP_RETURN_GENERATOR_r00] = "_GUARD_IP_RETURN_GENERATOR_r00", + [_GUARD_IP_RETURN_GENERATOR_r11] = "_GUARD_IP_RETURN_GENERATOR_r11", + [_GUARD_IP_RETURN_GENERATOR_r22] = "_GUARD_IP_RETURN_GENERATOR_r22", + [_GUARD_IP_RETURN_GENERATOR_r33] = "_GUARD_IP_RETURN_GENERATOR_r33", + [_GUARD_IP_RETURN_VALUE] = "_GUARD_IP_RETURN_VALUE", + [_GUARD_IP_RETURN_VALUE_r00] = "_GUARD_IP_RETURN_VALUE_r00", + [_GUARD_IP_RETURN_VALUE_r11] = "_GUARD_IP_RETURN_VALUE_r11", + [_GUARD_IP_RETURN_VALUE_r22] = "_GUARD_IP_RETURN_VALUE_r22", + [_GUARD_IP_RETURN_VALUE_r33] = "_GUARD_IP_RETURN_VALUE_r33", + [_GUARD_IP_YIELD_VALUE] = "_GUARD_IP_YIELD_VALUE", + [_GUARD_IP_YIELD_VALUE_r00] = "_GUARD_IP_YIELD_VALUE_r00", + [_GUARD_IP_YIELD_VALUE_r11] = "_GUARD_IP_YIELD_VALUE_r11", + [_GUARD_IP_YIELD_VALUE_r22] = "_GUARD_IP_YIELD_VALUE_r22", + [_GUARD_IP_YIELD_VALUE_r33] = "_GUARD_IP_YIELD_VALUE_r33", + [_GUARD_IP__PUSH_FRAME] = "_GUARD_IP__PUSH_FRAME", + [_GUARD_IP__PUSH_FRAME_r00] = "_GUARD_IP__PUSH_FRAME_r00", + [_GUARD_IP__PUSH_FRAME_r11] = "_GUARD_IP__PUSH_FRAME_r11", + [_GUARD_IP__PUSH_FRAME_r22] = "_GUARD_IP__PUSH_FRAME_r22", + [_GUARD_IP__PUSH_FRAME_r33] = "_GUARD_IP__PUSH_FRAME_r33", [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", + [_GUARD_IS_FALSE_POP_r00] = "_GUARD_IS_FALSE_POP_r00", + [_GUARD_IS_FALSE_POP_r10] = "_GUARD_IS_FALSE_POP_r10", + [_GUARD_IS_FALSE_POP_r21] = "_GUARD_IS_FALSE_POP_r21", + [_GUARD_IS_FALSE_POP_r32] = "_GUARD_IS_FALSE_POP_r32", [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", + [_GUARD_IS_NONE_POP_r00] = "_GUARD_IS_NONE_POP_r00", + [_GUARD_IS_NONE_POP_r10] = "_GUARD_IS_NONE_POP_r10", + [_GUARD_IS_NONE_POP_r21] = "_GUARD_IS_NONE_POP_r21", + [_GUARD_IS_NONE_POP_r32] = "_GUARD_IS_NONE_POP_r32", [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", + [_GUARD_IS_NOT_NONE_POP_r10] = "_GUARD_IS_NOT_NONE_POP_r10", [_GUARD_IS_TRUE_POP] = "_GUARD_IS_TRUE_POP", + [_GUARD_IS_TRUE_POP_r00] = "_GUARD_IS_TRUE_POP_r00", + [_GUARD_IS_TRUE_POP_r10] = "_GUARD_IS_TRUE_POP_r10", + [_GUARD_IS_TRUE_POP_r21] = "_GUARD_IS_TRUE_POP_r21", + [_GUARD_IS_TRUE_POP_r32] = "_GUARD_IS_TRUE_POP_r32", + [_GUARD_ITERATOR] = "_GUARD_ITERATOR", + [_GUARD_ITERATOR_r01] = "_GUARD_ITERATOR_r01", + [_GUARD_ITERATOR_r11] = "_GUARD_ITERATOR_r11", + [_GUARD_ITERATOR_r22] = "_GUARD_ITERATOR_r22", + [_GUARD_ITERATOR_r33] = "_GUARD_ITERATOR_r33", + [_GUARD_ITER_VIRTUAL] = "_GUARD_ITER_VIRTUAL", + [_GUARD_ITER_VIRTUAL_r01] = "_GUARD_ITER_VIRTUAL_r01", + [_GUARD_ITER_VIRTUAL_r11] = "_GUARD_ITER_VIRTUAL_r11", + [_GUARD_ITER_VIRTUAL_r22] = "_GUARD_ITER_VIRTUAL_r22", + [_GUARD_ITER_VIRTUAL_r33] = "_GUARD_ITER_VIRTUAL_r33", [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", + [_GUARD_KEYS_VERSION_r01] = "_GUARD_KEYS_VERSION_r01", + [_GUARD_KEYS_VERSION_r11] = "_GUARD_KEYS_VERSION_r11", + [_GUARD_KEYS_VERSION_r22] = "_GUARD_KEYS_VERSION_r22", + [_GUARD_KEYS_VERSION_r33] = "_GUARD_KEYS_VERSION_r33", + [_GUARD_LOAD_SUPER_ATTR_METHOD] = "_GUARD_LOAD_SUPER_ATTR_METHOD", + [_GUARD_LOAD_SUPER_ATTR_METHOD_r03] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r03", + [_GUARD_LOAD_SUPER_ATTR_METHOD_r13] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r13", + [_GUARD_LOAD_SUPER_ATTR_METHOD_r23] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r23", + [_GUARD_LOAD_SUPER_ATTR_METHOD_r33] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r33", + [_GUARD_NOS_ANY_DICT] = "_GUARD_NOS_ANY_DICT", + [_GUARD_NOS_ANY_DICT_r02] = "_GUARD_NOS_ANY_DICT_r02", + [_GUARD_NOS_ANY_DICT_r12] = "_GUARD_NOS_ANY_DICT_r12", + [_GUARD_NOS_ANY_DICT_r22] = "_GUARD_NOS_ANY_DICT_r22", + [_GUARD_NOS_ANY_DICT_r33] = "_GUARD_NOS_ANY_DICT_r33", + [_GUARD_NOS_COMPACT_ASCII] = "_GUARD_NOS_COMPACT_ASCII", + [_GUARD_NOS_COMPACT_ASCII_r02] = "_GUARD_NOS_COMPACT_ASCII_r02", + [_GUARD_NOS_COMPACT_ASCII_r12] = "_GUARD_NOS_COMPACT_ASCII_r12", + [_GUARD_NOS_COMPACT_ASCII_r22] = "_GUARD_NOS_COMPACT_ASCII_r22", + [_GUARD_NOS_COMPACT_ASCII_r33] = "_GUARD_NOS_COMPACT_ASCII_r33", [_GUARD_NOS_DICT] = "_GUARD_NOS_DICT", + [_GUARD_NOS_DICT_r02] = "_GUARD_NOS_DICT_r02", + [_GUARD_NOS_DICT_r12] = "_GUARD_NOS_DICT_r12", + [_GUARD_NOS_DICT_r22] = "_GUARD_NOS_DICT_r22", + [_GUARD_NOS_DICT_r33] = "_GUARD_NOS_DICT_r33", [_GUARD_NOS_FLOAT] = "_GUARD_NOS_FLOAT", + [_GUARD_NOS_FLOAT_r02] = "_GUARD_NOS_FLOAT_r02", + [_GUARD_NOS_FLOAT_r12] = "_GUARD_NOS_FLOAT_r12", + [_GUARD_NOS_FLOAT_r22] = "_GUARD_NOS_FLOAT_r22", + [_GUARD_NOS_FLOAT_r33] = "_GUARD_NOS_FLOAT_r33", [_GUARD_NOS_INT] = "_GUARD_NOS_INT", + [_GUARD_NOS_INT_r02] = "_GUARD_NOS_INT_r02", + [_GUARD_NOS_INT_r12] = "_GUARD_NOS_INT_r12", + [_GUARD_NOS_INT_r22] = "_GUARD_NOS_INT_r22", + [_GUARD_NOS_INT_r33] = "_GUARD_NOS_INT_r33", + [_GUARD_NOS_ITER_VIRTUAL] = "_GUARD_NOS_ITER_VIRTUAL", + [_GUARD_NOS_ITER_VIRTUAL_r02] = "_GUARD_NOS_ITER_VIRTUAL_r02", + [_GUARD_NOS_ITER_VIRTUAL_r12] = "_GUARD_NOS_ITER_VIRTUAL_r12", + [_GUARD_NOS_ITER_VIRTUAL_r22] = "_GUARD_NOS_ITER_VIRTUAL_r22", + [_GUARD_NOS_ITER_VIRTUAL_r33] = "_GUARD_NOS_ITER_VIRTUAL_r33", [_GUARD_NOS_LIST] = "_GUARD_NOS_LIST", + [_GUARD_NOS_LIST_r02] = "_GUARD_NOS_LIST_r02", + [_GUARD_NOS_LIST_r12] = "_GUARD_NOS_LIST_r12", + [_GUARD_NOS_LIST_r22] = "_GUARD_NOS_LIST_r22", + [_GUARD_NOS_LIST_r33] = "_GUARD_NOS_LIST_r33", [_GUARD_NOS_NOT_NULL] = "_GUARD_NOS_NOT_NULL", + [_GUARD_NOS_NOT_NULL_r02] = "_GUARD_NOS_NOT_NULL_r02", + [_GUARD_NOS_NOT_NULL_r12] = "_GUARD_NOS_NOT_NULL_r12", + [_GUARD_NOS_NOT_NULL_r22] = "_GUARD_NOS_NOT_NULL_r22", + [_GUARD_NOS_NOT_NULL_r33] = "_GUARD_NOS_NOT_NULL_r33", [_GUARD_NOS_NULL] = "_GUARD_NOS_NULL", + [_GUARD_NOS_NULL_r02] = "_GUARD_NOS_NULL_r02", + [_GUARD_NOS_NULL_r12] = "_GUARD_NOS_NULL_r12", + [_GUARD_NOS_NULL_r22] = "_GUARD_NOS_NULL_r22", + [_GUARD_NOS_NULL_r33] = "_GUARD_NOS_NULL_r33", [_GUARD_NOS_OVERFLOWED] = "_GUARD_NOS_OVERFLOWED", + [_GUARD_NOS_OVERFLOWED_r02] = "_GUARD_NOS_OVERFLOWED_r02", + [_GUARD_NOS_OVERFLOWED_r12] = "_GUARD_NOS_OVERFLOWED_r12", + [_GUARD_NOS_OVERFLOWED_r22] = "_GUARD_NOS_OVERFLOWED_r22", + [_GUARD_NOS_OVERFLOWED_r33] = "_GUARD_NOS_OVERFLOWED_r33", [_GUARD_NOS_TUPLE] = "_GUARD_NOS_TUPLE", + [_GUARD_NOS_TUPLE_r02] = "_GUARD_NOS_TUPLE_r02", + [_GUARD_NOS_TUPLE_r12] = "_GUARD_NOS_TUPLE_r12", + [_GUARD_NOS_TUPLE_r22] = "_GUARD_NOS_TUPLE_r22", + [_GUARD_NOS_TUPLE_r33] = "_GUARD_NOS_TUPLE_r33", + [_GUARD_NOS_TYPE_VERSION] = "_GUARD_NOS_TYPE_VERSION", + [_GUARD_NOS_TYPE_VERSION_r02] = "_GUARD_NOS_TYPE_VERSION_r02", + [_GUARD_NOS_TYPE_VERSION_r12] = "_GUARD_NOS_TYPE_VERSION_r12", + [_GUARD_NOS_TYPE_VERSION_r22] = "_GUARD_NOS_TYPE_VERSION_r22", + [_GUARD_NOS_TYPE_VERSION_r33] = "_GUARD_NOS_TYPE_VERSION_r33", [_GUARD_NOS_UNICODE] = "_GUARD_NOS_UNICODE", + [_GUARD_NOS_UNICODE_r02] = "_GUARD_NOS_UNICODE_r02", + [_GUARD_NOS_UNICODE_r12] = "_GUARD_NOS_UNICODE_r12", + [_GUARD_NOS_UNICODE_r22] = "_GUARD_NOS_UNICODE_r22", + [_GUARD_NOS_UNICODE_r33] = "_GUARD_NOS_UNICODE_r33", [_GUARD_NOT_EXHAUSTED_LIST] = "_GUARD_NOT_EXHAUSTED_LIST", + [_GUARD_NOT_EXHAUSTED_LIST_r02] = "_GUARD_NOT_EXHAUSTED_LIST_r02", + [_GUARD_NOT_EXHAUSTED_LIST_r12] = "_GUARD_NOT_EXHAUSTED_LIST_r12", + [_GUARD_NOT_EXHAUSTED_LIST_r22] = "_GUARD_NOT_EXHAUSTED_LIST_r22", + [_GUARD_NOT_EXHAUSTED_LIST_r33] = "_GUARD_NOT_EXHAUSTED_LIST_r33", [_GUARD_NOT_EXHAUSTED_RANGE] = "_GUARD_NOT_EXHAUSTED_RANGE", + [_GUARD_NOT_EXHAUSTED_RANGE_r02] = "_GUARD_NOT_EXHAUSTED_RANGE_r02", + [_GUARD_NOT_EXHAUSTED_RANGE_r12] = "_GUARD_NOT_EXHAUSTED_RANGE_r12", + [_GUARD_NOT_EXHAUSTED_RANGE_r22] = "_GUARD_NOT_EXHAUSTED_RANGE_r22", + [_GUARD_NOT_EXHAUSTED_RANGE_r33] = "_GUARD_NOT_EXHAUSTED_RANGE_r33", [_GUARD_NOT_EXHAUSTED_TUPLE] = "_GUARD_NOT_EXHAUSTED_TUPLE", + [_GUARD_NOT_EXHAUSTED_TUPLE_r02] = "_GUARD_NOT_EXHAUSTED_TUPLE_r02", + [_GUARD_NOT_EXHAUSTED_TUPLE_r12] = "_GUARD_NOT_EXHAUSTED_TUPLE_r12", + [_GUARD_NOT_EXHAUSTED_TUPLE_r22] = "_GUARD_NOT_EXHAUSTED_TUPLE_r22", + [_GUARD_NOT_EXHAUSTED_TUPLE_r33] = "_GUARD_NOT_EXHAUSTED_TUPLE_r33", [_GUARD_THIRD_NULL] = "_GUARD_THIRD_NULL", + [_GUARD_THIRD_NULL_r03] = "_GUARD_THIRD_NULL_r03", + [_GUARD_THIRD_NULL_r13] = "_GUARD_THIRD_NULL_r13", + [_GUARD_THIRD_NULL_r23] = "_GUARD_THIRD_NULL_r23", + [_GUARD_THIRD_NULL_r33] = "_GUARD_THIRD_NULL_r33", + [_GUARD_TOS_ANY_DICT] = "_GUARD_TOS_ANY_DICT", + [_GUARD_TOS_ANY_DICT_r01] = "_GUARD_TOS_ANY_DICT_r01", + [_GUARD_TOS_ANY_DICT_r11] = "_GUARD_TOS_ANY_DICT_r11", + [_GUARD_TOS_ANY_DICT_r22] = "_GUARD_TOS_ANY_DICT_r22", + [_GUARD_TOS_ANY_DICT_r33] = "_GUARD_TOS_ANY_DICT_r33", [_GUARD_TOS_ANY_SET] = "_GUARD_TOS_ANY_SET", + [_GUARD_TOS_ANY_SET_r01] = "_GUARD_TOS_ANY_SET_r01", + [_GUARD_TOS_ANY_SET_r11] = "_GUARD_TOS_ANY_SET_r11", + [_GUARD_TOS_ANY_SET_r22] = "_GUARD_TOS_ANY_SET_r22", + [_GUARD_TOS_ANY_SET_r33] = "_GUARD_TOS_ANY_SET_r33", [_GUARD_TOS_DICT] = "_GUARD_TOS_DICT", + [_GUARD_TOS_DICT_r01] = "_GUARD_TOS_DICT_r01", + [_GUARD_TOS_DICT_r11] = "_GUARD_TOS_DICT_r11", + [_GUARD_TOS_DICT_r22] = "_GUARD_TOS_DICT_r22", + [_GUARD_TOS_DICT_r33] = "_GUARD_TOS_DICT_r33", [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", + [_GUARD_TOS_FLOAT_r01] = "_GUARD_TOS_FLOAT_r01", + [_GUARD_TOS_FLOAT_r11] = "_GUARD_TOS_FLOAT_r11", + [_GUARD_TOS_FLOAT_r22] = "_GUARD_TOS_FLOAT_r22", + [_GUARD_TOS_FLOAT_r33] = "_GUARD_TOS_FLOAT_r33", + [_GUARD_TOS_FROZENDICT] = "_GUARD_TOS_FROZENDICT", + [_GUARD_TOS_FROZENDICT_r01] = "_GUARD_TOS_FROZENDICT_r01", + [_GUARD_TOS_FROZENDICT_r11] = "_GUARD_TOS_FROZENDICT_r11", + [_GUARD_TOS_FROZENDICT_r22] = "_GUARD_TOS_FROZENDICT_r22", + [_GUARD_TOS_FROZENDICT_r33] = "_GUARD_TOS_FROZENDICT_r33", + [_GUARD_TOS_FROZENSET] = "_GUARD_TOS_FROZENSET", + [_GUARD_TOS_FROZENSET_r01] = "_GUARD_TOS_FROZENSET_r01", + [_GUARD_TOS_FROZENSET_r11] = "_GUARD_TOS_FROZENSET_r11", + [_GUARD_TOS_FROZENSET_r22] = "_GUARD_TOS_FROZENSET_r22", + [_GUARD_TOS_FROZENSET_r33] = "_GUARD_TOS_FROZENSET_r33", [_GUARD_TOS_INT] = "_GUARD_TOS_INT", + [_GUARD_TOS_INT_r01] = "_GUARD_TOS_INT_r01", + [_GUARD_TOS_INT_r11] = "_GUARD_TOS_INT_r11", + [_GUARD_TOS_INT_r22] = "_GUARD_TOS_INT_r22", + [_GUARD_TOS_INT_r33] = "_GUARD_TOS_INT_r33", [_GUARD_TOS_LIST] = "_GUARD_TOS_LIST", + [_GUARD_TOS_LIST_r01] = "_GUARD_TOS_LIST_r01", + [_GUARD_TOS_LIST_r11] = "_GUARD_TOS_LIST_r11", + [_GUARD_TOS_LIST_r22] = "_GUARD_TOS_LIST_r22", + [_GUARD_TOS_LIST_r33] = "_GUARD_TOS_LIST_r33", [_GUARD_TOS_OVERFLOWED] = "_GUARD_TOS_OVERFLOWED", + [_GUARD_TOS_OVERFLOWED_r01] = "_GUARD_TOS_OVERFLOWED_r01", + [_GUARD_TOS_OVERFLOWED_r11] = "_GUARD_TOS_OVERFLOWED_r11", + [_GUARD_TOS_OVERFLOWED_r22] = "_GUARD_TOS_OVERFLOWED_r22", + [_GUARD_TOS_OVERFLOWED_r33] = "_GUARD_TOS_OVERFLOWED_r33", + [_GUARD_TOS_SET] = "_GUARD_TOS_SET", + [_GUARD_TOS_SET_r01] = "_GUARD_TOS_SET_r01", + [_GUARD_TOS_SET_r11] = "_GUARD_TOS_SET_r11", + [_GUARD_TOS_SET_r22] = "_GUARD_TOS_SET_r22", + [_GUARD_TOS_SET_r33] = "_GUARD_TOS_SET_r33", [_GUARD_TOS_SLICE] = "_GUARD_TOS_SLICE", + [_GUARD_TOS_SLICE_r01] = "_GUARD_TOS_SLICE_r01", + [_GUARD_TOS_SLICE_r11] = "_GUARD_TOS_SLICE_r11", + [_GUARD_TOS_SLICE_r22] = "_GUARD_TOS_SLICE_r22", + [_GUARD_TOS_SLICE_r33] = "_GUARD_TOS_SLICE_r33", [_GUARD_TOS_TUPLE] = "_GUARD_TOS_TUPLE", + [_GUARD_TOS_TUPLE_r01] = "_GUARD_TOS_TUPLE_r01", + [_GUARD_TOS_TUPLE_r11] = "_GUARD_TOS_TUPLE_r11", + [_GUARD_TOS_TUPLE_r22] = "_GUARD_TOS_TUPLE_r22", + [_GUARD_TOS_TUPLE_r33] = "_GUARD_TOS_TUPLE_r33", [_GUARD_TOS_UNICODE] = "_GUARD_TOS_UNICODE", + [_GUARD_TOS_UNICODE_r01] = "_GUARD_TOS_UNICODE_r01", + [_GUARD_TOS_UNICODE_r11] = "_GUARD_TOS_UNICODE_r11", + [_GUARD_TOS_UNICODE_r22] = "_GUARD_TOS_UNICODE_r22", + [_GUARD_TOS_UNICODE_r33] = "_GUARD_TOS_UNICODE_r33", + [_GUARD_TYPE] = "_GUARD_TYPE", + [_GUARD_TYPE_r01] = "_GUARD_TYPE_r01", + [_GUARD_TYPE_r11] = "_GUARD_TYPE_r11", + [_GUARD_TYPE_r22] = "_GUARD_TYPE_r22", + [_GUARD_TYPE_r33] = "_GUARD_TYPE_r33", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", - [_GUARD_TYPE_VERSION_AND_LOCK] = "_GUARD_TYPE_VERSION_AND_LOCK", + [_GUARD_TYPE_VERSION_r01] = "_GUARD_TYPE_VERSION_r01", + [_GUARD_TYPE_VERSION_r11] = "_GUARD_TYPE_VERSION_r11", + [_GUARD_TYPE_VERSION_r22] = "_GUARD_TYPE_VERSION_r22", + [_GUARD_TYPE_VERSION_r33] = "_GUARD_TYPE_VERSION_r33", + [_GUARD_TYPE_VERSION_LOCKED] = "_GUARD_TYPE_VERSION_LOCKED", + [_GUARD_TYPE_VERSION_LOCKED_r01] = "_GUARD_TYPE_VERSION_LOCKED_r01", + [_GUARD_TYPE_VERSION_LOCKED_r11] = "_GUARD_TYPE_VERSION_LOCKED_r11", + [_GUARD_TYPE_VERSION_LOCKED_r22] = "_GUARD_TYPE_VERSION_LOCKED_r22", + [_GUARD_TYPE_VERSION_LOCKED_r33] = "_GUARD_TYPE_VERSION_LOCKED_r33", [_HANDLE_PENDING_AND_DEOPT] = "_HANDLE_PENDING_AND_DEOPT", + [_HANDLE_PENDING_AND_DEOPT_r00] = "_HANDLE_PENDING_AND_DEOPT_r00", + [_HANDLE_PENDING_AND_DEOPT_r10] = "_HANDLE_PENDING_AND_DEOPT_r10", + [_HANDLE_PENDING_AND_DEOPT_r20] = "_HANDLE_PENDING_AND_DEOPT_r20", + [_HANDLE_PENDING_AND_DEOPT_r30] = "_HANDLE_PENDING_AND_DEOPT_r30", [_IMPORT_FROM] = "_IMPORT_FROM", + [_IMPORT_FROM_r12] = "_IMPORT_FROM_r12", [_IMPORT_NAME] = "_IMPORT_NAME", + [_IMPORT_NAME_r21] = "_IMPORT_NAME_r21", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00", [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", + [_INIT_CALL_PY_EXACT_ARGS_r01] = "_INIT_CALL_PY_EXACT_ARGS_r01", [_INIT_CALL_PY_EXACT_ARGS_0] = "_INIT_CALL_PY_EXACT_ARGS_0", + [_INIT_CALL_PY_EXACT_ARGS_0_r01] = "_INIT_CALL_PY_EXACT_ARGS_0_r01", [_INIT_CALL_PY_EXACT_ARGS_1] = "_INIT_CALL_PY_EXACT_ARGS_1", + [_INIT_CALL_PY_EXACT_ARGS_1_r01] = "_INIT_CALL_PY_EXACT_ARGS_1_r01", [_INIT_CALL_PY_EXACT_ARGS_2] = "_INIT_CALL_PY_EXACT_ARGS_2", + [_INIT_CALL_PY_EXACT_ARGS_2_r01] = "_INIT_CALL_PY_EXACT_ARGS_2_r01", [_INIT_CALL_PY_EXACT_ARGS_3] = "_INIT_CALL_PY_EXACT_ARGS_3", + [_INIT_CALL_PY_EXACT_ARGS_3_r01] = "_INIT_CALL_PY_EXACT_ARGS_3_r01", [_INIT_CALL_PY_EXACT_ARGS_4] = "_INIT_CALL_PY_EXACT_ARGS_4", + [_INIT_CALL_PY_EXACT_ARGS_4_r01] = "_INIT_CALL_PY_EXACT_ARGS_4_r01", [_INSERT_NULL] = "_INSERT_NULL", + [_INSERT_NULL_r10] = "_INSERT_NULL_r10", [_IS_NONE] = "_IS_NONE", + [_IS_NONE_r11] = "_IS_NONE_r11", [_IS_OP] = "_IS_OP", + [_IS_OP_r03] = "_IS_OP_r03", + [_IS_OP_r13] = "_IS_OP_r13", + [_IS_OP_r23] = "_IS_OP_r23", [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", + [_ITER_CHECK_LIST_r02] = "_ITER_CHECK_LIST_r02", + [_ITER_CHECK_LIST_r12] = "_ITER_CHECK_LIST_r12", + [_ITER_CHECK_LIST_r22] = "_ITER_CHECK_LIST_r22", + [_ITER_CHECK_LIST_r33] = "_ITER_CHECK_LIST_r33", [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", + [_ITER_CHECK_RANGE_r02] = "_ITER_CHECK_RANGE_r02", + [_ITER_CHECK_RANGE_r12] = "_ITER_CHECK_RANGE_r12", + [_ITER_CHECK_RANGE_r22] = "_ITER_CHECK_RANGE_r22", + [_ITER_CHECK_RANGE_r33] = "_ITER_CHECK_RANGE_r33", [_ITER_CHECK_TUPLE] = "_ITER_CHECK_TUPLE", + [_ITER_CHECK_TUPLE_r02] = "_ITER_CHECK_TUPLE_r02", + [_ITER_CHECK_TUPLE_r12] = "_ITER_CHECK_TUPLE_r12", + [_ITER_CHECK_TUPLE_r22] = "_ITER_CHECK_TUPLE_r22", + [_ITER_CHECK_TUPLE_r33] = "_ITER_CHECK_TUPLE_r33", [_ITER_NEXT_LIST_TIER_TWO] = "_ITER_NEXT_LIST_TIER_TWO", + [_ITER_NEXT_LIST_TIER_TWO_r23] = "_ITER_NEXT_LIST_TIER_TWO_r23", [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", + [_ITER_NEXT_RANGE_r03] = "_ITER_NEXT_RANGE_r03", + [_ITER_NEXT_RANGE_r13] = "_ITER_NEXT_RANGE_r13", + [_ITER_NEXT_RANGE_r23] = "_ITER_NEXT_RANGE_r23", [_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE", + [_ITER_NEXT_TUPLE_r03] = "_ITER_NEXT_TUPLE_r03", + [_ITER_NEXT_TUPLE_r13] = "_ITER_NEXT_TUPLE_r13", + [_ITER_NEXT_TUPLE_r23] = "_ITER_NEXT_TUPLE_r23", [_JUMP_TO_TOP] = "_JUMP_TO_TOP", + [_JUMP_TO_TOP_r00] = "_JUMP_TO_TOP_r00", [_LIST_APPEND] = "_LIST_APPEND", + [_LIST_APPEND_r10] = "_LIST_APPEND_r10", [_LIST_EXTEND] = "_LIST_EXTEND", + [_LIST_EXTEND_r11] = "_LIST_EXTEND_r11", [_LOAD_ATTR] = "_LOAD_ATTR", + [_LOAD_ATTR_r10] = "_LOAD_ATTR_r10", [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", + [_LOAD_ATTR_CLASS_r11] = "_LOAD_ATTR_CLASS_r11", + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME] = "_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME", + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11] = "_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11", [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", + [_LOAD_ATTR_INSTANCE_VALUE_r02] = "_LOAD_ATTR_INSTANCE_VALUE_r02", + [_LOAD_ATTR_INSTANCE_VALUE_r12] = "_LOAD_ATTR_INSTANCE_VALUE_r12", + [_LOAD_ATTR_INSTANCE_VALUE_r23] = "_LOAD_ATTR_INSTANCE_VALUE_r23", [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT", + [_LOAD_ATTR_METHOD_LAZY_DICT_r02] = "_LOAD_ATTR_METHOD_LAZY_DICT_r02", + [_LOAD_ATTR_METHOD_LAZY_DICT_r12] = "_LOAD_ATTR_METHOD_LAZY_DICT_r12", + [_LOAD_ATTR_METHOD_LAZY_DICT_r23] = "_LOAD_ATTR_METHOD_LAZY_DICT_r23", [_LOAD_ATTR_METHOD_NO_DICT] = "_LOAD_ATTR_METHOD_NO_DICT", + [_LOAD_ATTR_METHOD_NO_DICT_r02] = "_LOAD_ATTR_METHOD_NO_DICT_r02", + [_LOAD_ATTR_METHOD_NO_DICT_r12] = "_LOAD_ATTR_METHOD_NO_DICT_r12", + [_LOAD_ATTR_METHOD_NO_DICT_r23] = "_LOAD_ATTR_METHOD_NO_DICT_r23", [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", + [_LOAD_ATTR_METHOD_WITH_VALUES_r02] = "_LOAD_ATTR_METHOD_WITH_VALUES_r02", + [_LOAD_ATTR_METHOD_WITH_VALUES_r12] = "_LOAD_ATTR_METHOD_WITH_VALUES_r12", + [_LOAD_ATTR_METHOD_WITH_VALUES_r23] = "_LOAD_ATTR_METHOD_WITH_VALUES_r23", [_LOAD_ATTR_MODULE] = "_LOAD_ATTR_MODULE", + [_LOAD_ATTR_MODULE_r12] = "_LOAD_ATTR_MODULE_r12", [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11", [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11", [_LOAD_ATTR_PROPERTY_FRAME] = "_LOAD_ATTR_PROPERTY_FRAME", + [_LOAD_ATTR_PROPERTY_FRAME_r01] = "_LOAD_ATTR_PROPERTY_FRAME_r01", + [_LOAD_ATTR_PROPERTY_FRAME_r11] = "_LOAD_ATTR_PROPERTY_FRAME_r11", + [_LOAD_ATTR_PROPERTY_FRAME_r22] = "_LOAD_ATTR_PROPERTY_FRAME_r22", + [_LOAD_ATTR_PROPERTY_FRAME_r33] = "_LOAD_ATTR_PROPERTY_FRAME_r33", [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", + [_LOAD_ATTR_SLOT_r02] = "_LOAD_ATTR_SLOT_r02", + [_LOAD_ATTR_SLOT_r12] = "_LOAD_ATTR_SLOT_r12", + [_LOAD_ATTR_SLOT_r23] = "_LOAD_ATTR_SLOT_r23", [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", + [_LOAD_ATTR_WITH_HINT_r12] = "_LOAD_ATTR_WITH_HINT_r12", [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS", + [_LOAD_BUILD_CLASS_r01] = "_LOAD_BUILD_CLASS_r01", [_LOAD_COMMON_CONSTANT] = "_LOAD_COMMON_CONSTANT", + [_LOAD_COMMON_CONSTANT_r01] = "_LOAD_COMMON_CONSTANT_r01", + [_LOAD_COMMON_CONSTANT_r12] = "_LOAD_COMMON_CONSTANT_r12", + [_LOAD_COMMON_CONSTANT_r23] = "_LOAD_COMMON_CONSTANT_r23", [_LOAD_CONST] = "_LOAD_CONST", + [_LOAD_CONST_r01] = "_LOAD_CONST_r01", + [_LOAD_CONST_r12] = "_LOAD_CONST_r12", + [_LOAD_CONST_r23] = "_LOAD_CONST_r23", [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE", + [_LOAD_CONST_INLINE_r01] = "_LOAD_CONST_INLINE_r01", + [_LOAD_CONST_INLINE_r12] = "_LOAD_CONST_INLINE_r12", + [_LOAD_CONST_INLINE_r23] = "_LOAD_CONST_INLINE_r23", [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW", - [_LOAD_CONST_UNDER_INLINE] = "_LOAD_CONST_UNDER_INLINE", - [_LOAD_CONST_UNDER_INLINE_BORROW] = "_LOAD_CONST_UNDER_INLINE_BORROW", + [_LOAD_CONST_INLINE_BORROW_r01] = "_LOAD_CONST_INLINE_BORROW_r01", + [_LOAD_CONST_INLINE_BORROW_r12] = "_LOAD_CONST_INLINE_BORROW_r12", + [_LOAD_CONST_INLINE_BORROW_r23] = "_LOAD_CONST_INLINE_BORROW_r23", [_LOAD_DEREF] = "_LOAD_DEREF", + [_LOAD_DEREF_r01] = "_LOAD_DEREF_r01", [_LOAD_FAST] = "_LOAD_FAST", + [_LOAD_FAST_r01] = "_LOAD_FAST_r01", + [_LOAD_FAST_r12] = "_LOAD_FAST_r12", + [_LOAD_FAST_r23] = "_LOAD_FAST_r23", [_LOAD_FAST_0] = "_LOAD_FAST_0", + [_LOAD_FAST_0_r01] = "_LOAD_FAST_0_r01", + [_LOAD_FAST_0_r12] = "_LOAD_FAST_0_r12", + [_LOAD_FAST_0_r23] = "_LOAD_FAST_0_r23", [_LOAD_FAST_1] = "_LOAD_FAST_1", + [_LOAD_FAST_1_r01] = "_LOAD_FAST_1_r01", + [_LOAD_FAST_1_r12] = "_LOAD_FAST_1_r12", + [_LOAD_FAST_1_r23] = "_LOAD_FAST_1_r23", [_LOAD_FAST_2] = "_LOAD_FAST_2", + [_LOAD_FAST_2_r01] = "_LOAD_FAST_2_r01", + [_LOAD_FAST_2_r12] = "_LOAD_FAST_2_r12", + [_LOAD_FAST_2_r23] = "_LOAD_FAST_2_r23", [_LOAD_FAST_3] = "_LOAD_FAST_3", + [_LOAD_FAST_3_r01] = "_LOAD_FAST_3_r01", + [_LOAD_FAST_3_r12] = "_LOAD_FAST_3_r12", + [_LOAD_FAST_3_r23] = "_LOAD_FAST_3_r23", [_LOAD_FAST_4] = "_LOAD_FAST_4", + [_LOAD_FAST_4_r01] = "_LOAD_FAST_4_r01", + [_LOAD_FAST_4_r12] = "_LOAD_FAST_4_r12", + [_LOAD_FAST_4_r23] = "_LOAD_FAST_4_r23", [_LOAD_FAST_5] = "_LOAD_FAST_5", + [_LOAD_FAST_5_r01] = "_LOAD_FAST_5_r01", + [_LOAD_FAST_5_r12] = "_LOAD_FAST_5_r12", + [_LOAD_FAST_5_r23] = "_LOAD_FAST_5_r23", [_LOAD_FAST_6] = "_LOAD_FAST_6", + [_LOAD_FAST_6_r01] = "_LOAD_FAST_6_r01", + [_LOAD_FAST_6_r12] = "_LOAD_FAST_6_r12", + [_LOAD_FAST_6_r23] = "_LOAD_FAST_6_r23", [_LOAD_FAST_7] = "_LOAD_FAST_7", + [_LOAD_FAST_7_r01] = "_LOAD_FAST_7_r01", + [_LOAD_FAST_7_r12] = "_LOAD_FAST_7_r12", + [_LOAD_FAST_7_r23] = "_LOAD_FAST_7_r23", [_LOAD_FAST_AND_CLEAR] = "_LOAD_FAST_AND_CLEAR", + [_LOAD_FAST_AND_CLEAR_r01] = "_LOAD_FAST_AND_CLEAR_r01", + [_LOAD_FAST_AND_CLEAR_r12] = "_LOAD_FAST_AND_CLEAR_r12", + [_LOAD_FAST_AND_CLEAR_r23] = "_LOAD_FAST_AND_CLEAR_r23", [_LOAD_FAST_BORROW] = "_LOAD_FAST_BORROW", + [_LOAD_FAST_BORROW_r01] = "_LOAD_FAST_BORROW_r01", + [_LOAD_FAST_BORROW_r12] = "_LOAD_FAST_BORROW_r12", + [_LOAD_FAST_BORROW_r23] = "_LOAD_FAST_BORROW_r23", [_LOAD_FAST_BORROW_0] = "_LOAD_FAST_BORROW_0", + [_LOAD_FAST_BORROW_0_r01] = "_LOAD_FAST_BORROW_0_r01", + [_LOAD_FAST_BORROW_0_r12] = "_LOAD_FAST_BORROW_0_r12", + [_LOAD_FAST_BORROW_0_r23] = "_LOAD_FAST_BORROW_0_r23", [_LOAD_FAST_BORROW_1] = "_LOAD_FAST_BORROW_1", + [_LOAD_FAST_BORROW_1_r01] = "_LOAD_FAST_BORROW_1_r01", + [_LOAD_FAST_BORROW_1_r12] = "_LOAD_FAST_BORROW_1_r12", + [_LOAD_FAST_BORROW_1_r23] = "_LOAD_FAST_BORROW_1_r23", [_LOAD_FAST_BORROW_2] = "_LOAD_FAST_BORROW_2", + [_LOAD_FAST_BORROW_2_r01] = "_LOAD_FAST_BORROW_2_r01", + [_LOAD_FAST_BORROW_2_r12] = "_LOAD_FAST_BORROW_2_r12", + [_LOAD_FAST_BORROW_2_r23] = "_LOAD_FAST_BORROW_2_r23", [_LOAD_FAST_BORROW_3] = "_LOAD_FAST_BORROW_3", + [_LOAD_FAST_BORROW_3_r01] = "_LOAD_FAST_BORROW_3_r01", + [_LOAD_FAST_BORROW_3_r12] = "_LOAD_FAST_BORROW_3_r12", + [_LOAD_FAST_BORROW_3_r23] = "_LOAD_FAST_BORROW_3_r23", [_LOAD_FAST_BORROW_4] = "_LOAD_FAST_BORROW_4", + [_LOAD_FAST_BORROW_4_r01] = "_LOAD_FAST_BORROW_4_r01", + [_LOAD_FAST_BORROW_4_r12] = "_LOAD_FAST_BORROW_4_r12", + [_LOAD_FAST_BORROW_4_r23] = "_LOAD_FAST_BORROW_4_r23", [_LOAD_FAST_BORROW_5] = "_LOAD_FAST_BORROW_5", + [_LOAD_FAST_BORROW_5_r01] = "_LOAD_FAST_BORROW_5_r01", + [_LOAD_FAST_BORROW_5_r12] = "_LOAD_FAST_BORROW_5_r12", + [_LOAD_FAST_BORROW_5_r23] = "_LOAD_FAST_BORROW_5_r23", [_LOAD_FAST_BORROW_6] = "_LOAD_FAST_BORROW_6", + [_LOAD_FAST_BORROW_6_r01] = "_LOAD_FAST_BORROW_6_r01", + [_LOAD_FAST_BORROW_6_r12] = "_LOAD_FAST_BORROW_6_r12", + [_LOAD_FAST_BORROW_6_r23] = "_LOAD_FAST_BORROW_6_r23", [_LOAD_FAST_BORROW_7] = "_LOAD_FAST_BORROW_7", - [_LOAD_FAST_BORROW_LOAD_FAST_BORROW] = "_LOAD_FAST_BORROW_LOAD_FAST_BORROW", + [_LOAD_FAST_BORROW_7_r01] = "_LOAD_FAST_BORROW_7_r01", + [_LOAD_FAST_BORROW_7_r12] = "_LOAD_FAST_BORROW_7_r12", + [_LOAD_FAST_BORROW_7_r23] = "_LOAD_FAST_BORROW_7_r23", [_LOAD_FAST_CHECK] = "_LOAD_FAST_CHECK", - [_LOAD_FAST_LOAD_FAST] = "_LOAD_FAST_LOAD_FAST", + [_LOAD_FAST_CHECK_r01] = "_LOAD_FAST_CHECK_r01", + [_LOAD_FAST_CHECK_r12] = "_LOAD_FAST_CHECK_r12", + [_LOAD_FAST_CHECK_r23] = "_LOAD_FAST_CHECK_r23", [_LOAD_FROM_DICT_OR_DEREF] = "_LOAD_FROM_DICT_OR_DEREF", + [_LOAD_FROM_DICT_OR_DEREF_r11] = "_LOAD_FROM_DICT_OR_DEREF_r11", [_LOAD_GLOBAL] = "_LOAD_GLOBAL", + [_LOAD_GLOBAL_r00] = "_LOAD_GLOBAL_r00", [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", + [_LOAD_GLOBAL_BUILTINS_r01] = "_LOAD_GLOBAL_BUILTINS_r01", [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", + [_LOAD_GLOBAL_MODULE_r01] = "_LOAD_GLOBAL_MODULE_r01", [_LOAD_LOCALS] = "_LOAD_LOCALS", + [_LOAD_LOCALS_r01] = "_LOAD_LOCALS_r01", + [_LOAD_LOCALS_r12] = "_LOAD_LOCALS_r12", + [_LOAD_LOCALS_r23] = "_LOAD_LOCALS_r23", [_LOAD_NAME] = "_LOAD_NAME", + [_LOAD_NAME_r01] = "_LOAD_NAME_r01", [_LOAD_SMALL_INT] = "_LOAD_SMALL_INT", + [_LOAD_SMALL_INT_r01] = "_LOAD_SMALL_INT_r01", + [_LOAD_SMALL_INT_r12] = "_LOAD_SMALL_INT_r12", + [_LOAD_SMALL_INT_r23] = "_LOAD_SMALL_INT_r23", [_LOAD_SMALL_INT_0] = "_LOAD_SMALL_INT_0", + [_LOAD_SMALL_INT_0_r01] = "_LOAD_SMALL_INT_0_r01", + [_LOAD_SMALL_INT_0_r12] = "_LOAD_SMALL_INT_0_r12", + [_LOAD_SMALL_INT_0_r23] = "_LOAD_SMALL_INT_0_r23", [_LOAD_SMALL_INT_1] = "_LOAD_SMALL_INT_1", + [_LOAD_SMALL_INT_1_r01] = "_LOAD_SMALL_INT_1_r01", + [_LOAD_SMALL_INT_1_r12] = "_LOAD_SMALL_INT_1_r12", + [_LOAD_SMALL_INT_1_r23] = "_LOAD_SMALL_INT_1_r23", [_LOAD_SMALL_INT_2] = "_LOAD_SMALL_INT_2", + [_LOAD_SMALL_INT_2_r01] = "_LOAD_SMALL_INT_2_r01", + [_LOAD_SMALL_INT_2_r12] = "_LOAD_SMALL_INT_2_r12", + [_LOAD_SMALL_INT_2_r23] = "_LOAD_SMALL_INT_2_r23", [_LOAD_SMALL_INT_3] = "_LOAD_SMALL_INT_3", + [_LOAD_SMALL_INT_3_r01] = "_LOAD_SMALL_INT_3_r01", + [_LOAD_SMALL_INT_3_r12] = "_LOAD_SMALL_INT_3_r12", + [_LOAD_SMALL_INT_3_r23] = "_LOAD_SMALL_INT_3_r23", [_LOAD_SPECIAL] = "_LOAD_SPECIAL", + [_LOAD_SPECIAL_r00] = "_LOAD_SPECIAL_r00", [_LOAD_SUPER_ATTR_ATTR] = "_LOAD_SUPER_ATTR_ATTR", + [_LOAD_SUPER_ATTR_ATTR_r31] = "_LOAD_SUPER_ATTR_ATTR_r31", [_LOAD_SUPER_ATTR_METHOD] = "_LOAD_SUPER_ATTR_METHOD", + [_LOAD_SUPER_ATTR_METHOD_r32] = "_LOAD_SUPER_ATTR_METHOD_r32", + [_LOCK_OBJECT] = "_LOCK_OBJECT", + [_LOCK_OBJECT_r01] = "_LOCK_OBJECT_r01", + [_LOCK_OBJECT_r11] = "_LOCK_OBJECT_r11", + [_LOCK_OBJECT_r22] = "_LOCK_OBJECT_r22", + [_LOCK_OBJECT_r33] = "_LOCK_OBJECT_r33", [_MAKE_CALLARGS_A_TUPLE] = "_MAKE_CALLARGS_A_TUPLE", + [_MAKE_CALLARGS_A_TUPLE_r33] = "_MAKE_CALLARGS_A_TUPLE_r33", [_MAKE_CELL] = "_MAKE_CELL", + [_MAKE_CELL_r00] = "_MAKE_CELL_r00", [_MAKE_FUNCTION] = "_MAKE_FUNCTION", + [_MAKE_FUNCTION_r12] = "_MAKE_FUNCTION_r12", + [_MAKE_HEAP_SAFE] = "_MAKE_HEAP_SAFE", + [_MAKE_HEAP_SAFE_r01] = "_MAKE_HEAP_SAFE_r01", + [_MAKE_HEAP_SAFE_r11] = "_MAKE_HEAP_SAFE_r11", + [_MAKE_HEAP_SAFE_r22] = "_MAKE_HEAP_SAFE_r22", + [_MAKE_HEAP_SAFE_r33] = "_MAKE_HEAP_SAFE_r33", [_MAKE_WARM] = "_MAKE_WARM", + [_MAKE_WARM_r00] = "_MAKE_WARM_r00", + [_MAKE_WARM_r11] = "_MAKE_WARM_r11", + [_MAKE_WARM_r22] = "_MAKE_WARM_r22", + [_MAKE_WARM_r33] = "_MAKE_WARM_r33", [_MAP_ADD] = "_MAP_ADD", + [_MAP_ADD_r20] = "_MAP_ADD_r20", [_MATCH_CLASS] = "_MATCH_CLASS", + [_MATCH_CLASS_r33] = "_MATCH_CLASS_r33", [_MATCH_KEYS] = "_MATCH_KEYS", + [_MATCH_KEYS_r23] = "_MATCH_KEYS_r23", [_MATCH_MAPPING] = "_MATCH_MAPPING", + [_MATCH_MAPPING_r02] = "_MATCH_MAPPING_r02", + [_MATCH_MAPPING_r12] = "_MATCH_MAPPING_r12", + [_MATCH_MAPPING_r23] = "_MATCH_MAPPING_r23", [_MATCH_SEQUENCE] = "_MATCH_SEQUENCE", + [_MATCH_SEQUENCE_r02] = "_MATCH_SEQUENCE_r02", + [_MATCH_SEQUENCE_r12] = "_MATCH_SEQUENCE_r12", + [_MATCH_SEQUENCE_r23] = "_MATCH_SEQUENCE_r23", [_MAYBE_EXPAND_METHOD] = "_MAYBE_EXPAND_METHOD", + [_MAYBE_EXPAND_METHOD_r00] = "_MAYBE_EXPAND_METHOD_r00", [_MAYBE_EXPAND_METHOD_KW] = "_MAYBE_EXPAND_METHOD_KW", + [_MAYBE_EXPAND_METHOD_KW_r11] = "_MAYBE_EXPAND_METHOD_KW_r11", [_NOP] = "_NOP", - [_POP_CALL] = "_POP_CALL", - [_POP_CALL_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_LOAD_CONST_INLINE_BORROW", - [_POP_CALL_ONE] = "_POP_CALL_ONE", - [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW", - [_POP_CALL_TWO] = "_POP_CALL_TWO", - [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW", + [_NOP_r00] = "_NOP_r00", + [_NOP_r11] = "_NOP_r11", + [_NOP_r22] = "_NOP_r22", + [_NOP_r33] = "_NOP_r33", [_POP_EXCEPT] = "_POP_EXCEPT", + [_POP_EXCEPT_r10] = "_POP_EXCEPT_r10", [_POP_ITER] = "_POP_ITER", + [_POP_ITER_r20] = "_POP_ITER_r20", [_POP_TOP] = "_POP_TOP", + [_POP_TOP_r10] = "_POP_TOP_r10", [_POP_TOP_FLOAT] = "_POP_TOP_FLOAT", + [_POP_TOP_FLOAT_r00] = "_POP_TOP_FLOAT_r00", + [_POP_TOP_FLOAT_r10] = "_POP_TOP_FLOAT_r10", + [_POP_TOP_FLOAT_r21] = "_POP_TOP_FLOAT_r21", + [_POP_TOP_FLOAT_r32] = "_POP_TOP_FLOAT_r32", [_POP_TOP_INT] = "_POP_TOP_INT", - [_POP_TOP_LOAD_CONST_INLINE] = "_POP_TOP_LOAD_CONST_INLINE", - [_POP_TOP_LOAD_CONST_INLINE_BORROW] = "_POP_TOP_LOAD_CONST_INLINE_BORROW", + [_POP_TOP_INT_r00] = "_POP_TOP_INT_r00", + [_POP_TOP_INT_r10] = "_POP_TOP_INT_r10", + [_POP_TOP_INT_r21] = "_POP_TOP_INT_r21", + [_POP_TOP_INT_r32] = "_POP_TOP_INT_r32", [_POP_TOP_NOP] = "_POP_TOP_NOP", + [_POP_TOP_NOP_r00] = "_POP_TOP_NOP_r00", + [_POP_TOP_NOP_r10] = "_POP_TOP_NOP_r10", + [_POP_TOP_NOP_r21] = "_POP_TOP_NOP_r21", + [_POP_TOP_NOP_r32] = "_POP_TOP_NOP_r32", + [_POP_TOP_OPARG] = "_POP_TOP_OPARG", + [_POP_TOP_OPARG_r00] = "_POP_TOP_OPARG_r00", [_POP_TOP_UNICODE] = "_POP_TOP_UNICODE", - [_POP_TWO] = "_POP_TWO", - [_POP_TWO_LOAD_CONST_INLINE_BORROW] = "_POP_TWO_LOAD_CONST_INLINE_BORROW", + [_POP_TOP_UNICODE_r00] = "_POP_TOP_UNICODE_r00", + [_POP_TOP_UNICODE_r10] = "_POP_TOP_UNICODE_r10", + [_POP_TOP_UNICODE_r21] = "_POP_TOP_UNICODE_r21", + [_POP_TOP_UNICODE_r32] = "_POP_TOP_UNICODE_r32", [_PUSH_EXC_INFO] = "_PUSH_EXC_INFO", + [_PUSH_EXC_INFO_r02] = "_PUSH_EXC_INFO_r02", + [_PUSH_EXC_INFO_r12] = "_PUSH_EXC_INFO_r12", + [_PUSH_EXC_INFO_r23] = "_PUSH_EXC_INFO_r23", [_PUSH_FRAME] = "_PUSH_FRAME", + [_PUSH_FRAME_r10] = "_PUSH_FRAME_r10", [_PUSH_NULL] = "_PUSH_NULL", + [_PUSH_NULL_r01] = "_PUSH_NULL_r01", + [_PUSH_NULL_r12] = "_PUSH_NULL_r12", + [_PUSH_NULL_r23] = "_PUSH_NULL_r23", [_PUSH_NULL_CONDITIONAL] = "_PUSH_NULL_CONDITIONAL", + [_PUSH_NULL_CONDITIONAL_r00] = "_PUSH_NULL_CONDITIONAL_r00", + [_PUSH_TAGGED_ZERO] = "_PUSH_TAGGED_ZERO", + [_PUSH_TAGGED_ZERO_r01] = "_PUSH_TAGGED_ZERO_r01", + [_PUSH_TAGGED_ZERO_r12] = "_PUSH_TAGGED_ZERO_r12", + [_PUSH_TAGGED_ZERO_r23] = "_PUSH_TAGGED_ZERO_r23", + [_PY_FRAME_EX] = "_PY_FRAME_EX", + [_PY_FRAME_EX_r31] = "_PY_FRAME_EX_r31", [_PY_FRAME_GENERAL] = "_PY_FRAME_GENERAL", + [_PY_FRAME_GENERAL_r01] = "_PY_FRAME_GENERAL_r01", [_PY_FRAME_KW] = "_PY_FRAME_KW", + [_PY_FRAME_KW_r11] = "_PY_FRAME_KW_r11", + [_RECORD_3OS_GEN_FUNC] = "_RECORD_3OS_GEN_FUNC", + [_RECORD_4OS] = "_RECORD_4OS", + [_RECORD_BOUND_METHOD] = "_RECORD_BOUND_METHOD", + [_RECORD_CALLABLE] = "_RECORD_CALLABLE", + [_RECORD_CALLABLE_KW] = "_RECORD_CALLABLE_KW", + [_RECORD_CODE] = "_RECORD_CODE", + [_RECORD_NOS] = "_RECORD_NOS", + [_RECORD_NOS_GEN_FUNC] = "_RECORD_NOS_GEN_FUNC", + [_RECORD_NOS_TYPE] = "_RECORD_NOS_TYPE", + [_RECORD_TOS] = "_RECORD_TOS", + [_RECORD_TOS_TYPE] = "_RECORD_TOS_TYPE", [_REPLACE_WITH_TRUE] = "_REPLACE_WITH_TRUE", + [_REPLACE_WITH_TRUE_r02] = "_REPLACE_WITH_TRUE_r02", + [_REPLACE_WITH_TRUE_r12] = "_REPLACE_WITH_TRUE_r12", + [_REPLACE_WITH_TRUE_r23] = "_REPLACE_WITH_TRUE_r23", [_RESUME_CHECK] = "_RESUME_CHECK", + [_RESUME_CHECK_r00] = "_RESUME_CHECK_r00", + [_RESUME_CHECK_r11] = "_RESUME_CHECK_r11", + [_RESUME_CHECK_r22] = "_RESUME_CHECK_r22", + [_RESUME_CHECK_r33] = "_RESUME_CHECK_r33", [_RETURN_GENERATOR] = "_RETURN_GENERATOR", + [_RETURN_GENERATOR_r01] = "_RETURN_GENERATOR_r01", [_RETURN_VALUE] = "_RETURN_VALUE", + [_RETURN_VALUE_r11] = "_RETURN_VALUE_r11", + [_RROT_3] = "_RROT_3", + [_RROT_3_r03] = "_RROT_3_r03", + [_RROT_3_r13] = "_RROT_3_r13", + [_RROT_3_r23] = "_RROT_3_r23", + [_RROT_3_r33] = "_RROT_3_r33", [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", + [_SAVE_RETURN_OFFSET_r00] = "_SAVE_RETURN_OFFSET_r00", + [_SAVE_RETURN_OFFSET_r11] = "_SAVE_RETURN_OFFSET_r11", + [_SAVE_RETURN_OFFSET_r22] = "_SAVE_RETURN_OFFSET_r22", + [_SAVE_RETURN_OFFSET_r33] = "_SAVE_RETURN_OFFSET_r33", [_SEND_GEN_FRAME] = "_SEND_GEN_FRAME", + [_SEND_GEN_FRAME_r33] = "_SEND_GEN_FRAME_r33", [_SETUP_ANNOTATIONS] = "_SETUP_ANNOTATIONS", + [_SETUP_ANNOTATIONS_r00] = "_SETUP_ANNOTATIONS_r00", [_SET_ADD] = "_SET_ADD", + [_SET_ADD_r10] = "_SET_ADD_r10", [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", + [_SET_FUNCTION_ATTRIBUTE_r01] = "_SET_FUNCTION_ATTRIBUTE_r01", + [_SET_FUNCTION_ATTRIBUTE_r11] = "_SET_FUNCTION_ATTRIBUTE_r11", + [_SET_FUNCTION_ATTRIBUTE_r21] = "_SET_FUNCTION_ATTRIBUTE_r21", + [_SET_FUNCTION_ATTRIBUTE_r32] = "_SET_FUNCTION_ATTRIBUTE_r32", [_SET_IP] = "_SET_IP", + [_SET_IP_r00] = "_SET_IP_r00", + [_SET_IP_r11] = "_SET_IP_r11", + [_SET_IP_r22] = "_SET_IP_r22", + [_SET_IP_r33] = "_SET_IP_r33", [_SET_UPDATE] = "_SET_UPDATE", + [_SET_UPDATE_r11] = "_SET_UPDATE_r11", + [_SPILL_OR_RELOAD] = "_SPILL_OR_RELOAD", + [_SPILL_OR_RELOAD_r01] = "_SPILL_OR_RELOAD_r01", + [_SPILL_OR_RELOAD_r02] = "_SPILL_OR_RELOAD_r02", + [_SPILL_OR_RELOAD_r03] = "_SPILL_OR_RELOAD_r03", + [_SPILL_OR_RELOAD_r10] = "_SPILL_OR_RELOAD_r10", + [_SPILL_OR_RELOAD_r12] = "_SPILL_OR_RELOAD_r12", + [_SPILL_OR_RELOAD_r13] = "_SPILL_OR_RELOAD_r13", + [_SPILL_OR_RELOAD_r20] = "_SPILL_OR_RELOAD_r20", + [_SPILL_OR_RELOAD_r21] = "_SPILL_OR_RELOAD_r21", + [_SPILL_OR_RELOAD_r23] = "_SPILL_OR_RELOAD_r23", + [_SPILL_OR_RELOAD_r30] = "_SPILL_OR_RELOAD_r30", + [_SPILL_OR_RELOAD_r31] = "_SPILL_OR_RELOAD_r31", + [_SPILL_OR_RELOAD_r32] = "_SPILL_OR_RELOAD_r32", [_START_EXECUTOR] = "_START_EXECUTOR", + [_START_EXECUTOR_r00] = "_START_EXECUTOR_r00", [_STORE_ATTR] = "_STORE_ATTR", + [_STORE_ATTR_r20] = "_STORE_ATTR_r20", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", + [_STORE_ATTR_INSTANCE_VALUE_r21] = "_STORE_ATTR_INSTANCE_VALUE_r21", [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", + [_STORE_ATTR_SLOT_r21] = "_STORE_ATTR_SLOT_r21", [_STORE_ATTR_WITH_HINT] = "_STORE_ATTR_WITH_HINT", + [_STORE_ATTR_WITH_HINT_r21] = "_STORE_ATTR_WITH_HINT_r21", [_STORE_DEREF] = "_STORE_DEREF", - [_STORE_FAST] = "_STORE_FAST", - [_STORE_FAST_0] = "_STORE_FAST_0", - [_STORE_FAST_1] = "_STORE_FAST_1", - [_STORE_FAST_2] = "_STORE_FAST_2", - [_STORE_FAST_3] = "_STORE_FAST_3", - [_STORE_FAST_4] = "_STORE_FAST_4", - [_STORE_FAST_5] = "_STORE_FAST_5", - [_STORE_FAST_6] = "_STORE_FAST_6", - [_STORE_FAST_7] = "_STORE_FAST_7", - [_STORE_FAST_LOAD_FAST] = "_STORE_FAST_LOAD_FAST", - [_STORE_FAST_STORE_FAST] = "_STORE_FAST_STORE_FAST", + [_STORE_DEREF_r10] = "_STORE_DEREF_r10", [_STORE_GLOBAL] = "_STORE_GLOBAL", + [_STORE_GLOBAL_r10] = "_STORE_GLOBAL_r10", [_STORE_NAME] = "_STORE_NAME", + [_STORE_NAME_r10] = "_STORE_NAME_r10", [_STORE_SLICE] = "_STORE_SLICE", + [_STORE_SLICE_r30] = "_STORE_SLICE_r30", [_STORE_SUBSCR] = "_STORE_SUBSCR", + [_STORE_SUBSCR_r30] = "_STORE_SUBSCR_r30", [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", + [_STORE_SUBSCR_DICT_r31] = "_STORE_SUBSCR_DICT_r31", + [_STORE_SUBSCR_DICT_KNOWN_HASH] = "_STORE_SUBSCR_DICT_KNOWN_HASH", + [_STORE_SUBSCR_DICT_KNOWN_HASH_r31] = "_STORE_SUBSCR_DICT_KNOWN_HASH_r31", [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", + [_STORE_SUBSCR_LIST_INT_r32] = "_STORE_SUBSCR_LIST_INT_r32", [_SWAP] = "_SWAP", + [_SWAP_r11] = "_SWAP_r11", [_SWAP_2] = "_SWAP_2", + [_SWAP_2_r02] = "_SWAP_2_r02", + [_SWAP_2_r12] = "_SWAP_2_r12", + [_SWAP_2_r22] = "_SWAP_2_r22", + [_SWAP_2_r33] = "_SWAP_2_r33", [_SWAP_3] = "_SWAP_3", + [_SWAP_3_r03] = "_SWAP_3_r03", + [_SWAP_3_r13] = "_SWAP_3_r13", + [_SWAP_3_r23] = "_SWAP_3_r23", + [_SWAP_3_r33] = "_SWAP_3_r33", + [_SWAP_FAST] = "_SWAP_FAST", + [_SWAP_FAST_r01] = "_SWAP_FAST_r01", + [_SWAP_FAST_r11] = "_SWAP_FAST_r11", + [_SWAP_FAST_r22] = "_SWAP_FAST_r22", + [_SWAP_FAST_r33] = "_SWAP_FAST_r33", + [_SWAP_FAST_0] = "_SWAP_FAST_0", + [_SWAP_FAST_0_r01] = "_SWAP_FAST_0_r01", + [_SWAP_FAST_0_r11] = "_SWAP_FAST_0_r11", + [_SWAP_FAST_0_r22] = "_SWAP_FAST_0_r22", + [_SWAP_FAST_0_r33] = "_SWAP_FAST_0_r33", + [_SWAP_FAST_1] = "_SWAP_FAST_1", + [_SWAP_FAST_1_r01] = "_SWAP_FAST_1_r01", + [_SWAP_FAST_1_r11] = "_SWAP_FAST_1_r11", + [_SWAP_FAST_1_r22] = "_SWAP_FAST_1_r22", + [_SWAP_FAST_1_r33] = "_SWAP_FAST_1_r33", + [_SWAP_FAST_2] = "_SWAP_FAST_2", + [_SWAP_FAST_2_r01] = "_SWAP_FAST_2_r01", + [_SWAP_FAST_2_r11] = "_SWAP_FAST_2_r11", + [_SWAP_FAST_2_r22] = "_SWAP_FAST_2_r22", + [_SWAP_FAST_2_r33] = "_SWAP_FAST_2_r33", + [_SWAP_FAST_3] = "_SWAP_FAST_3", + [_SWAP_FAST_3_r01] = "_SWAP_FAST_3_r01", + [_SWAP_FAST_3_r11] = "_SWAP_FAST_3_r11", + [_SWAP_FAST_3_r22] = "_SWAP_FAST_3_r22", + [_SWAP_FAST_3_r33] = "_SWAP_FAST_3_r33", + [_SWAP_FAST_4] = "_SWAP_FAST_4", + [_SWAP_FAST_4_r01] = "_SWAP_FAST_4_r01", + [_SWAP_FAST_4_r11] = "_SWAP_FAST_4_r11", + [_SWAP_FAST_4_r22] = "_SWAP_FAST_4_r22", + [_SWAP_FAST_4_r33] = "_SWAP_FAST_4_r33", + [_SWAP_FAST_5] = "_SWAP_FAST_5", + [_SWAP_FAST_5_r01] = "_SWAP_FAST_5_r01", + [_SWAP_FAST_5_r11] = "_SWAP_FAST_5_r11", + [_SWAP_FAST_5_r22] = "_SWAP_FAST_5_r22", + [_SWAP_FAST_5_r33] = "_SWAP_FAST_5_r33", + [_SWAP_FAST_6] = "_SWAP_FAST_6", + [_SWAP_FAST_6_r01] = "_SWAP_FAST_6_r01", + [_SWAP_FAST_6_r11] = "_SWAP_FAST_6_r11", + [_SWAP_FAST_6_r22] = "_SWAP_FAST_6_r22", + [_SWAP_FAST_6_r33] = "_SWAP_FAST_6_r33", + [_SWAP_FAST_7] = "_SWAP_FAST_7", + [_SWAP_FAST_7_r01] = "_SWAP_FAST_7_r01", + [_SWAP_FAST_7_r11] = "_SWAP_FAST_7_r11", + [_SWAP_FAST_7_r22] = "_SWAP_FAST_7_r22", + [_SWAP_FAST_7_r33] = "_SWAP_FAST_7_r33", [_TIER2_RESUME_CHECK] = "_TIER2_RESUME_CHECK", + [_TIER2_RESUME_CHECK_r00] = "_TIER2_RESUME_CHECK_r00", + [_TIER2_RESUME_CHECK_r11] = "_TIER2_RESUME_CHECK_r11", + [_TIER2_RESUME_CHECK_r22] = "_TIER2_RESUME_CHECK_r22", + [_TIER2_RESUME_CHECK_r33] = "_TIER2_RESUME_CHECK_r33", [_TO_BOOL] = "_TO_BOOL", + [_TO_BOOL_r11] = "_TO_BOOL_r11", [_TO_BOOL_BOOL] = "_TO_BOOL_BOOL", + [_TO_BOOL_BOOL_r01] = "_TO_BOOL_BOOL_r01", + [_TO_BOOL_BOOL_r11] = "_TO_BOOL_BOOL_r11", + [_TO_BOOL_BOOL_r22] = "_TO_BOOL_BOOL_r22", + [_TO_BOOL_BOOL_r33] = "_TO_BOOL_BOOL_r33", [_TO_BOOL_INT] = "_TO_BOOL_INT", + [_TO_BOOL_INT_r02] = "_TO_BOOL_INT_r02", + [_TO_BOOL_INT_r12] = "_TO_BOOL_INT_r12", + [_TO_BOOL_INT_r23] = "_TO_BOOL_INT_r23", [_TO_BOOL_LIST] = "_TO_BOOL_LIST", + [_TO_BOOL_LIST_r02] = "_TO_BOOL_LIST_r02", + [_TO_BOOL_LIST_r12] = "_TO_BOOL_LIST_r12", + [_TO_BOOL_LIST_r23] = "_TO_BOOL_LIST_r23", [_TO_BOOL_NONE] = "_TO_BOOL_NONE", + [_TO_BOOL_NONE_r01] = "_TO_BOOL_NONE_r01", + [_TO_BOOL_NONE_r11] = "_TO_BOOL_NONE_r11", + [_TO_BOOL_NONE_r22] = "_TO_BOOL_NONE_r22", + [_TO_BOOL_NONE_r33] = "_TO_BOOL_NONE_r33", [_TO_BOOL_STR] = "_TO_BOOL_STR", + [_TO_BOOL_STR_r02] = "_TO_BOOL_STR_r02", + [_TO_BOOL_STR_r12] = "_TO_BOOL_STR_r12", + [_TO_BOOL_STR_r23] = "_TO_BOOL_STR_r23", [_UNARY_INVERT] = "_UNARY_INVERT", + [_UNARY_INVERT_r12] = "_UNARY_INVERT_r12", [_UNARY_NEGATIVE] = "_UNARY_NEGATIVE", + [_UNARY_NEGATIVE_r12] = "_UNARY_NEGATIVE_r12", + [_UNARY_NEGATIVE_FLOAT_INPLACE] = "_UNARY_NEGATIVE_FLOAT_INPLACE", + [_UNARY_NEGATIVE_FLOAT_INPLACE_r02] = "_UNARY_NEGATIVE_FLOAT_INPLACE_r02", + [_UNARY_NEGATIVE_FLOAT_INPLACE_r12] = "_UNARY_NEGATIVE_FLOAT_INPLACE_r12", + [_UNARY_NEGATIVE_FLOAT_INPLACE_r23] = "_UNARY_NEGATIVE_FLOAT_INPLACE_r23", [_UNARY_NOT] = "_UNARY_NOT", + [_UNARY_NOT_r01] = "_UNARY_NOT_r01", + [_UNARY_NOT_r11] = "_UNARY_NOT_r11", + [_UNARY_NOT_r22] = "_UNARY_NOT_r22", + [_UNARY_NOT_r33] = "_UNARY_NOT_r33", [_UNPACK_EX] = "_UNPACK_EX", + [_UNPACK_EX_r10] = "_UNPACK_EX_r10", [_UNPACK_SEQUENCE] = "_UNPACK_SEQUENCE", + [_UNPACK_SEQUENCE_r10] = "_UNPACK_SEQUENCE_r10", [_UNPACK_SEQUENCE_LIST] = "_UNPACK_SEQUENCE_LIST", + [_UNPACK_SEQUENCE_LIST_r10] = "_UNPACK_SEQUENCE_LIST_r10", [_UNPACK_SEQUENCE_TUPLE] = "_UNPACK_SEQUENCE_TUPLE", + [_UNPACK_SEQUENCE_TUPLE_r10] = "_UNPACK_SEQUENCE_TUPLE_r10", [_UNPACK_SEQUENCE_TWO_TUPLE] = "_UNPACK_SEQUENCE_TWO_TUPLE", + [_UNPACK_SEQUENCE_TWO_TUPLE_r12] = "_UNPACK_SEQUENCE_TWO_TUPLE_r12", + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE] = "_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE", + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03] = "_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03", + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13] = "_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13", + [_UNPACK_SEQUENCE_UNIQUE_TUPLE] = "_UNPACK_SEQUENCE_UNIQUE_TUPLE", + [_UNPACK_SEQUENCE_UNIQUE_TUPLE_r10] = "_UNPACK_SEQUENCE_UNIQUE_TUPLE_r10", + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE] = "_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE", + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02] = "_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02", + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12] = "_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12", + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23] = "_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23", [_WITH_EXCEPT_START] = "_WITH_EXCEPT_START", + [_WITH_EXCEPT_START_r33] = "_WITH_EXCEPT_START_r33", [_YIELD_VALUE] = "_YIELD_VALUE", + [_YIELD_VALUE_r11] = "_YIELD_VALUE_r11", }; int _PyUop_num_popped(int opcode, int oparg) { @@ -721,10 +6151,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_FAST_AND_CLEAR: return 0; - case _LOAD_FAST_LOAD_FAST: - return 0; - case _LOAD_FAST_BORROW_LOAD_FAST_BORROW: - return 0; case _LOAD_CONST: return 0; case _LOAD_SMALL_INT_0: @@ -737,28 +6163,24 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_SMALL_INT: return 0; - case _STORE_FAST_0: + case _SWAP_FAST_0: return 1; - case _STORE_FAST_1: + case _SWAP_FAST_1: return 1; - case _STORE_FAST_2: + case _SWAP_FAST_2: return 1; - case _STORE_FAST_3: + case _SWAP_FAST_3: return 1; - case _STORE_FAST_4: + case _SWAP_FAST_4: return 1; - case _STORE_FAST_5: + case _SWAP_FAST_5: return 1; - case _STORE_FAST_6: + case _SWAP_FAST_6: return 1; - case _STORE_FAST_7: + case _SWAP_FAST_7: return 1; - case _STORE_FAST: + case _SWAP_FAST: return 1; - case _STORE_FAST_LOAD_FAST: - return 1; - case _STORE_FAST_STORE_FAST: - return 2; case _POP_TOP: return 1; case _POP_TOP_NOP: @@ -769,8 +6191,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _POP_TOP_UNICODE: return 1; - case _POP_TWO: - return 2; + case _POP_TOP_OPARG: + return oparg; case _PUSH_NULL: return 0; case _END_FOR: @@ -778,9 +6200,11 @@ int _PyUop_num_popped(int opcode, int oparg) case _POP_ITER: return 2; case _END_SEND: - return 2; + return 3; case _UNARY_NEGATIVE: return 1; + case _UNARY_NEGATIVE_FLOAT_INPLACE: + return 1; case _UNARY_NOT: return 1; case _TO_BOOL: @@ -799,6 +6223,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _TO_BOOL_NONE: return 1; + case _GUARD_NOS_COMPACT_ASCII: + return 0; case _GUARD_NOS_UNICODE: return 0; case _GUARD_TOS_UNICODE: @@ -823,6 +6249,18 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBTRACT_INT: return 2; + case _BINARY_OP_ADD_INT_INPLACE: + return 2; + case _BINARY_OP_SUBTRACT_INT_INPLACE: + return 2; + case _BINARY_OP_MULTIPLY_INT_INPLACE: + return 2; + case _BINARY_OP_ADD_INT_INPLACE_RIGHT: + return 2; + case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT: + return 2; + case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT: + return 2; case _GUARD_NOS_FLOAT: return 0; case _GUARD_TOS_FLOAT: @@ -833,16 +6271,32 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBTRACT_FLOAT: return 2; - case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: + case _BINARY_OP_ADD_FLOAT_INPLACE: return 2; - case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE: return 2; - case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE: + return 2; + case _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT: + return 2; + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT: + return 2; + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT: + return 2; + case _BINARY_OP_TRUEDIV_FLOAT: + return 2; + case _BINARY_OP_TRUEDIV_FLOAT_INPLACE: + return 2; + case _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT: return 2; case _BINARY_OP_ADD_UNICODE: return 2; case _BINARY_OP_INPLACE_ADD_UNICODE: return 2; + case _GUARD_BINARY_OP_EXTEND_LHS: + return 0; + case _GUARD_BINARY_OP_EXTEND_RHS: + return 0; case _GUARD_BINARY_OP_EXTEND: return 0; case _BINARY_OP_EXTEND: @@ -857,16 +6311,28 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBSCR_STR_INT: return 2; + case _BINARY_OP_SUBSCR_USTR_INT: + return 2; case _GUARD_NOS_TUPLE: return 0; case _GUARD_TOS_TUPLE: return 0; + case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS: + return 0; case _BINARY_OP_SUBSCR_TUPLE_INT: return 2; case _GUARD_NOS_DICT: return 0; + case _GUARD_NOS_ANY_DICT: + return 0; + case _GUARD_TOS_ANY_DICT: + return 0; case _GUARD_TOS_DICT: return 0; + case _GUARD_TOS_FROZENDICT: + return 0; + case _BINARY_OP_SUBSCR_DICT_KNOWN_HASH: + return 2; case _BINARY_OP_SUBSCR_DICT: return 2; case _BINARY_OP_SUBSCR_CHECK_FUNC: @@ -883,12 +6349,16 @@ int _PyUop_num_popped(int opcode, int oparg) return 3; case _STORE_SUBSCR_DICT: return 3; + case _STORE_SUBSCR_DICT_KNOWN_HASH: + return 3; case _DELETE_SUBSCR: return 2; case _CALL_INTRINSIC_1: return 1; case _CALL_INTRINSIC_2: return 2; + case _MAKE_HEAP_SAFE: + return 0; case _RETURN_VALUE: return 1; case _GET_AITER: @@ -915,8 +6385,14 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _UNPACK_SEQUENCE_TWO_TUPLE: return 1; + case _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE: + return 1; + case _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE: + return 1; case _UNPACK_SEQUENCE_TUPLE: return 1; + case _UNPACK_SEQUENCE_UNIQUE_TUPLE: + return 1; case _UNPACK_SEQUENCE_LIST: return 1; case _UNPACK_EX: @@ -985,13 +6461,19 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _LOAD_SUPER_ATTR_ATTR: return 3; + case _GUARD_NOS_TYPE_VERSION: + return 0; + case _GUARD_LOAD_SUPER_ATTR_METHOD: + return 0; case _LOAD_SUPER_ATTR_METHOD: return 3; case _LOAD_ATTR: return 1; case _GUARD_TYPE_VERSION: return 0; - case _GUARD_TYPE_VERSION_AND_LOCK: + case _GUARD_TYPE_VERSION_LOCKED: + return 0; + case _GUARD_TYPE: return 0; case _CHECK_MANAGED_OBJECT_HAS_VALUES: return 0; @@ -1009,10 +6491,14 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _LOAD_ATTR_PROPERTY_FRAME: return 1; + case _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME: + return 1; case _GUARD_DORV_NO_DICT: return 0; case _STORE_ATTR_INSTANCE_VALUE: return 2; + case _LOCK_OBJECT: + return 0; case _STORE_ATTR_WITH_HINT: return 2; case _STORE_ATTR_SLOT: @@ -1031,6 +6517,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _GUARD_TOS_ANY_SET: return 0; + case _GUARD_TOS_SET: + return 0; + case _GUARD_TOS_FROZENSET: + return 0; case _CONTAINS_OP_SET: return 2; case _CONTAINS_OP_DICT: @@ -1057,10 +6547,20 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GET_ITER: return 1; - case _GET_YIELD_FROM_ITER: + case _GUARD_ITERATOR: + return 0; + case _GUARD_ITER_VIRTUAL: + return 0; + case _PUSH_TAGGED_ZERO: + return 0; + case _GET_ITER_TRAD: return 1; case _FOR_ITER_TIER_TWO: return 0; + case _GUARD_NOS_ITER_VIRTUAL: + return 0; + case _FOR_ITER_VIRTUAL_TIER_TWO: + return 0; case _ITER_CHECK_LIST: return 0; case _GUARD_NOT_EXHAUSTED_LIST: @@ -1165,20 +6665,30 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _CALL_TUPLE_1: return 3; - case _CHECK_AND_ALLOCATE_OBJECT: + case _CHECK_OBJECT: + return 0; + case _ALLOCATE_OBJECT: return 0; case _CREATE_INIT_FRAME: return 2 + oparg; case _EXIT_INIT_CHECK: return 1; + case _GUARD_CALLABLE_BUILTIN_CLASS: + return 0; case _CALL_BUILTIN_CLASS: - return 2 + oparg; + return 0; + case _GUARD_CALLABLE_BUILTIN_O: + return 0; case _CALL_BUILTIN_O: return 2 + oparg; + case _GUARD_CALLABLE_BUILTIN_FAST: + return 0; case _CALL_BUILTIN_FAST: - return 2 + oparg; + return 0; + case _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS: + return 0; case _CALL_BUILTIN_FAST_WITH_KEYWORDS: - return 2 + oparg; + return 0; case _GUARD_CALLABLE_LEN: return 0; case _CALL_LEN: @@ -1191,14 +6701,32 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _CALL_LIST_APPEND: return 3; + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_O: + return 0; case _CALL_METHOD_DESCRIPTOR_O: return 2 + oparg; + case _CHECK_RECURSION_LIMIT: + return 0; + case _CALL_METHOD_DESCRIPTOR_O_INLINE: + return 1 + oparg; + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: + return 0; case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: - return 2 + oparg; + return 0; + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE: + return 0; + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS: + return 0; case _CALL_METHOD_DESCRIPTOR_NOARGS: return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE: + return 1 + oparg; + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST: + return 0; case _CALL_METHOD_DESCRIPTOR_FAST: - return 2 + oparg; + return 0; + case _CALL_METHOD_DESCRIPTOR_FAST_INLINE: + return 0; case _MAYBE_EXPAND_METHOD_KW: return 0; case _PY_FRAME_KW: @@ -1215,6 +6743,14 @@ int _PyUop_num_popped(int opcode, int oparg) return 3 + oparg; case _MAKE_CALLARGS_A_TUPLE: return 0; + case _CHECK_IS_PY_CALLABLE_EX: + return 0; + case _PY_FRAME_EX: + return 4; + case _CHECK_IS_NOT_PY_CALLABLE_EX: + return 0; + case _CALL_FUNCTION_EX_NON_PY_GENERAL: + return 4; case _MAKE_FUNCTION: return 1; case _SET_FUNCTION_ATTRIBUTE: @@ -1249,6 +6785,26 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _GUARD_IS_FALSE_POP: return 1; + case _GUARD_BIT_IS_SET_POP_4: + return 1; + case _GUARD_BIT_IS_SET_POP_5: + return 1; + case _GUARD_BIT_IS_SET_POP_6: + return 1; + case _GUARD_BIT_IS_SET_POP_7: + return 1; + case _GUARD_BIT_IS_SET_POP: + return 1; + case _GUARD_BIT_IS_UNSET_POP_4: + return 1; + case _GUARD_BIT_IS_UNSET_POP_5: + return 1; + case _GUARD_BIT_IS_UNSET_POP_6: + return 1; + case _GUARD_BIT_IS_UNSET_POP_7: + return 1; + case _GUARD_BIT_IS_UNSET_POP: + return 1; case _GUARD_IS_NONE_POP: return 1; case _GUARD_IS_NOT_NONE_POP: @@ -1263,35 +6819,15 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _EXIT_TRACE: return 0; + case _DYNAMIC_EXIT: + return 0; case _CHECK_VALIDITY: return 0; case _LOAD_CONST_INLINE: return 0; - case _POP_TOP_LOAD_CONST_INLINE: - return 1; case _LOAD_CONST_INLINE_BORROW: return 0; - case _POP_CALL: - return 2; - case _POP_CALL_ONE: - return 3; - case _POP_CALL_TWO: - return 4; - case _POP_TOP_LOAD_CONST_INLINE_BORROW: - return 1; - case _POP_TWO_LOAD_CONST_INLINE_BORROW: - return 2; - case _POP_CALL_LOAD_CONST_INLINE_BORROW: - return 2; - case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: - return 3; - case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: - return 4; - case _LOAD_CONST_UNDER_INLINE: - return 1; - case _LOAD_CONST_UNDER_INLINE_BORROW: - return 1; - case _CHECK_FUNCTION: + case _RROT_3: return 0; case _START_EXECUTOR: return 0; @@ -1305,10 +6841,52 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _ERROR_POP_N: return 0; + case _SPILL_OR_RELOAD: + return 0; case _TIER2_RESUME_CHECK: return 0; case _COLD_EXIT: return 0; + case _COLD_DYNAMIC_EXIT: + return 0; + case _GUARD_CODE_VERSION__PUSH_FRAME: + return 0; + case _GUARD_CODE_VERSION_YIELD_VALUE: + return 0; + case _GUARD_CODE_VERSION_RETURN_VALUE: + return 0; + case _GUARD_CODE_VERSION_RETURN_GENERATOR: + return 0; + case _GUARD_IP__PUSH_FRAME: + return 0; + case _GUARD_IP_YIELD_VALUE: + return 0; + case _GUARD_IP_RETURN_VALUE: + return 0; + case _GUARD_IP_RETURN_GENERATOR: + return 0; + case _RECORD_TOS: + return 0; + case _RECORD_TOS_TYPE: + return 0; + case _RECORD_NOS: + return 0; + case _RECORD_NOS_TYPE: + return 0; + case _RECORD_NOS_GEN_FUNC: + return 0; + case _RECORD_3OS_GEN_FUNC: + return 0; + case _RECORD_4OS: + return 0; + case _RECORD_CALLABLE: + return 0; + case _RECORD_CALLABLE_KW: + return 0; + case _RECORD_BOUND_METHOD: + return 0; + case _RECORD_CODE: + return 0; default: return -1; } diff --git a/Include/longobject.h b/Include/longobject.h index 19f06977036..38673bc1878 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -166,6 +166,44 @@ PyAPI_FUNC(PyObject *) PyLong_FromString(const char *, char **, int); PyAPI_FUNC(unsigned long) PyOS_strtoul(const char *, char **, int); PyAPI_FUNC(long) PyOS_strtol(const char *, char **, int); +/* --- Import/Export API -------------------------------------------------- */ + +typedef struct PyLongLayout { + uint8_t bits_per_digit; + uint8_t digit_size; + int8_t digits_order; + int8_t digit_endianness; +} PyLongLayout; + +PyAPI_FUNC(const PyLongLayout*) PyLong_GetNativeLayout(void); + +typedef struct PyLongExport { + int64_t value; + uint8_t negative; + Py_ssize_t ndigits; + const void *digits; + // Member used internally, must not be used for other purpose. + Py_uintptr_t _reserved; +} PyLongExport; + +PyAPI_FUNC(int) PyLong_Export( + PyObject *obj, + PyLongExport *export_long); +PyAPI_FUNC(void) PyLong_FreeExport( + PyLongExport *export_long); + + +/* --- PyLongWriter API --------------------------------------------------- */ + +typedef struct PyLongWriter PyLongWriter; + +PyAPI_FUNC(PyLongWriter*) PyLongWriter_Create( + int negative, + Py_ssize_t ndigits, + void **digits); +PyAPI_FUNC(PyObject*) PyLongWriter_Finish(PyLongWriter *writer); +PyAPI_FUNC(void) PyLongWriter_Discard(PyLongWriter *writer); + #ifndef Py_LIMITED_API # define Py_CPYTHON_LONGOBJECT_H # include "cpython/longobject.h" diff --git a/Include/marshal.h b/Include/marshal.h index f773587bdd0..2ccb112b40c 100644 --- a/Include/marshal.h +++ b/Include/marshal.h @@ -1,31 +1,18 @@ - /* Interface for marshal.c */ #ifndef Py_MARSHAL_H #define Py_MARSHAL_H -#ifndef Py_LIMITED_API - #ifdef __cplusplus extern "C" { #endif -PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, - Py_ssize_t); -PyAPI_FUNC(PyObject *) PyMarshal_WriteObjectToString(PyObject *, int); - -#define Py_MARSHAL_VERSION 5 - -PyAPI_FUNC(long) PyMarshal_ReadLongFromFile(FILE *); -PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *); -PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromFile(FILE *); -PyAPI_FUNC(PyObject *) PyMarshal_ReadLastObjectFromFile(FILE *); - -PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int); -PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int); +#ifndef Py_LIMITED_API +# define _Py_CPYTHON_MARSHAL_H +# include "cpython/marshal.h" +# undef _Py_CPYTHON_MARSHAL_H +#endif #ifdef __cplusplus } #endif - -#endif /* Py_LIMITED_API */ #endif /* !Py_MARSHAL_H */ diff --git a/Include/modsupport.h b/Include/modsupport.h index 094b9ff0e5c..cb47ad8cd27 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -132,7 +132,7 @@ PyAPI_FUNC(int) PyABIInfo_Check(PyABIInfo *info, const char *module_name); ) \ ///////////////////////////////////////////////////////// -#define _PyABIInfo_DEFAULT() { \ +#define _PyABIInfo_DEFAULT { \ 1, 0, \ PyABIInfo_DEFAULT_FLAGS, \ PY_VERSION_HEX, \ diff --git a/Include/moduleobject.h b/Include/moduleobject.h index e3afac0a343..c2fb1f85165 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -83,11 +83,19 @@ struct PyModuleDef_Slot { #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) # define Py_mod_abi 5 +# define Py_mod_name 6 +# define Py_mod_doc 7 +# define Py_mod_state_size 8 +# define Py_mod_methods 9 +# define Py_mod_state_traverse 10 +# define Py_mod_state_clear 11 +# define Py_mod_state_free 12 +# define Py_mod_token 13 #endif #ifndef Py_LIMITED_API -#define _Py_mod_LAST_SLOT 5 +#define _Py_mod_LAST_SLOT 13 #endif #endif /* New in 3.5 */ @@ -105,10 +113,21 @@ struct PyModuleDef_Slot { # define Py_MOD_GIL_NOT_USED ((void *)1) #endif -#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) +#if !defined(Py_LIMITED_API) +# if defined(Py_GIL_DISABLED) PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil); +# endif #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(PyObject *) PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, + PyObject *spec); +PyAPI_FUNC(int) PyModule_Exec(PyObject *module); +PyAPI_FUNC(int) PyModule_GetStateSize(PyObject *module, Py_ssize_t *result); +PyAPI_FUNC(int) PyModule_GetToken(PyObject *module, void **result); +PyAPI_FUNC(void*) PyModule_GetState_DuringGC(PyObject*); +PyAPI_FUNC(int) PyModule_GetToken_DuringGC(PyObject *module, void **result); +#endif #ifndef _Py_OPAQUE_PYOBJECT struct PyModuleDef { diff --git a/Include/monitoring.h b/Include/monitoring.h deleted file mode 100644 index 985f7f230e4..00000000000 --- a/Include/monitoring.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef Py_MONITORING_H -#define Py_MONITORING_H -#ifdef __cplusplus -extern "C" { -#endif - -// There is currently no limited API for monitoring - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_MONITORING_H -# include "cpython/monitoring.h" -# undef Py_CPYTHON_MONITORING_H -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_MONITORING_H */ diff --git a/Include/object.h b/Include/object.h index 064904b733d..d51132be1a6 100644 --- a/Include/object.h +++ b/Include/object.h @@ -71,6 +71,8 @@ whose size is determined when the object is allocated. * * Statically allocated objects might be shared between * interpreters, so must be marked as immortal. + * + * Before changing this, see the check in PyModuleDef_Init(). */ #if defined(Py_GIL_DISABLED) #define PyObject_HEAD_INIT(type) \ @@ -125,7 +127,7 @@ whose size is determined when the object is allocated. struct _object { _Py_ANONYMOUS union { #if SIZEOF_VOID_P > 4 - PY_INT64_T ob_refcnt_full; /* This field is needed for efficient initialization with Clang on ARM */ + int64_t ob_refcnt_full; /* This field is needed for efficient initialization with Clang on ARM */ struct { # if PY_BIG_ENDIAN uint16_t ob_flags; @@ -138,12 +140,12 @@ struct _object { # endif }; #else - Py_ssize_t ob_refcnt; + Py_ssize_t ob_refcnt; // part of stable ABI; do not change #endif _Py_ALIGNED_DEF(_PyObject_MIN_ALIGNMENT, char) _aligner; }; - PyTypeObject *ob_type; + PyTypeObject *ob_type; // part of stable ABI; do not change }; #else // Objects that are not owned by any thread use a thread id (tid) of zero. @@ -171,7 +173,7 @@ struct _object { #ifndef _Py_OPAQUE_PYOBJECT struct PyVarObject { PyObject ob_base; - Py_ssize_t ob_size; /* Number of items in variable part */ + Py_ssize_t ob_size; // Number of items in variable part. Part of stable ABI }; #endif typedef struct PyVarObject PyVarObject; @@ -184,135 +186,72 @@ typedef struct PyVarObject PyVarObject; PyAPI_FUNC(int) Py_Is(PyObject *x, PyObject *y); #define Py_Is(x, y) ((x) == (y)) -#if defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) -PyAPI_FUNC(uintptr_t) _Py_GetThreadLocal_Addr(void); - -static inline uintptr_t -_Py_ThreadId(void) -{ - uintptr_t tid; -#if defined(_MSC_VER) && defined(_M_X64) - tid = __readgsqword(48); -#elif defined(_MSC_VER) && defined(_M_IX86) - tid = __readfsdword(24); -#elif defined(_MSC_VER) && defined(_M_ARM64) - tid = __getReg(18); -#elif defined(__MINGW32__) && defined(_M_X64) - tid = __readgsqword(48); -#elif defined(__MINGW32__) && defined(_M_IX86) - tid = __readfsdword(24); -#elif defined(__MINGW32__) && defined(_M_ARM64) - tid = __getReg(18); -#elif defined(__i386__) - __asm__("movl %%gs:0, %0" : "=r" (tid)); // 32-bit always uses GS -#elif defined(__MACH__) && defined(__x86_64__) - __asm__("movq %%gs:0, %0" : "=r" (tid)); // x86_64 macOSX uses GS -#elif defined(__x86_64__) - __asm__("movq %%fs:0, %0" : "=r" (tid)); // x86_64 Linux, BSD uses FS -#elif defined(__arm__) && __ARM_ARCH >= 7 - __asm__ ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tid)); -#elif defined(__aarch64__) && defined(__APPLE__) - __asm__ ("mrs %0, tpidrro_el0" : "=r" (tid)); -#elif defined(__aarch64__) - __asm__ ("mrs %0, tpidr_el0" : "=r" (tid)); -#elif defined(__powerpc64__) - #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) - tid = (uintptr_t)__builtin_thread_pointer(); - #else - // r13 is reserved for use as system thread ID by the Power 64-bit ABI. - register uintptr_t tp __asm__ ("r13"); - __asm__("" : "=r" (tp)); - tid = tp; - #endif -#elif defined(__powerpc__) - #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) - tid = (uintptr_t)__builtin_thread_pointer(); - #else - // r2 is reserved for use as system thread ID by the Power 32-bit ABI. - register uintptr_t tp __asm__ ("r2"); - __asm__ ("" : "=r" (tp)); - tid = tp; - #endif -#elif defined(__s390__) && defined(__GNUC__) - // Both GCC and Clang have supported __builtin_thread_pointer - // for s390 from long time ago. - tid = (uintptr_t)__builtin_thread_pointer(); -#elif defined(__riscv) - #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) - tid = (uintptr_t)__builtin_thread_pointer(); - #else - // tp is Thread Pointer provided by the RISC-V ABI. - __asm__ ("mv %0, tp" : "=r" (tid)); - #endif -#else - // Fallback to a portable implementation if we do not have a faster - // platform-specific implementation. - tid = _Py_GetThreadLocal_Addr(); -#endif - return tid; -} - -static inline Py_ALWAYS_INLINE int -_Py_IsOwnedByCurrentThread(PyObject *ob) -{ -#ifdef _Py_THREAD_SANITIZER - return _Py_atomic_load_uintptr_relaxed(&ob->ob_tid) == _Py_ThreadId(); -#else - return ob->ob_tid == _Py_ThreadId(); -#endif -} -#endif - -// Py_TYPE() implementation for the stable ABI -PyAPI_FUNC(PyTypeObject*) Py_TYPE(PyObject *ob); - -#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030e0000 - // Stable ABI implements Py_TYPE() as a function call - // on limited C API version 3.14 and newer. -#else - static inline PyTypeObject* _Py_TYPE(PyObject *ob) - { - return ob->ob_type; - } - #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 - # define Py_TYPE(ob) _Py_TYPE(_PyObject_CAST(ob)) - #else - # define Py_TYPE(ob) _Py_TYPE(ob) - #endif -#endif - PyAPI_DATA(PyTypeObject) PyLong_Type; PyAPI_DATA(PyTypeObject) PyBool_Type; +/* Definitions for the stable ABI */ +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 14) +PyAPI_FUNC(PyTypeObject*) Py_TYPE(PyObject *ob); +#endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(Py_ssize_t) Py_SIZE(PyObject *ob); +PyAPI_FUNC(int) Py_IS_TYPE(PyObject *ob, PyTypeObject *type); +PyAPI_FUNC(void) Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size); +#endif + #ifndef _Py_OPAQUE_PYOBJECT + +static inline void +Py_SET_TYPE(PyObject *ob, PyTypeObject *type) +{ + ob->ob_type = type; +} + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < _Py_PACK_VERSION(3, 11) +// Non-limited API & limited API 3.11 & below: use static inline functions and +// use _PyObject_CAST so that users don't need their own casts +# define Py_TYPE(ob) _Py_TYPE_impl(_PyObject_CAST(ob)) +# define Py_SIZE(ob) _Py_SIZE_impl(_PyObject_CAST(ob)) +# define Py_IS_TYPE(ob, type) _Py_IS_TYPE_impl(_PyObject_CAST(ob), (type)) +# define Py_SET_SIZE(ob, size) _Py_SET_SIZE_impl(_PyVarObject_CAST(ob), (size)) +# define Py_SET_TYPE(ob, type) Py_SET_TYPE(_PyObject_CAST(ob), type) +#elif Py_LIMITED_API+0 < _Py_PACK_VERSION(3, 15) +// Limited API 3.11-3.14: use static inline functions, without casts +# define Py_SIZE(ob) _Py_SIZE_impl(ob) +# define Py_IS_TYPE(ob, type) _Py_IS_TYPE_impl((ob), (type)) +# define Py_SET_SIZE(ob, size) _Py_SET_SIZE_impl((ob), (size)) +# if Py_LIMITED_API+0 < _Py_PACK_VERSION(3, 14) +// Py_TYPE() is static inline only on Limited API 3.13 and below +# define Py_TYPE(ob) _Py_TYPE_impl(ob) +# endif +#else +// Limited API 3.15+: use function calls +#endif + +static inline +PyTypeObject* _Py_TYPE_impl(PyObject *ob) +{ + return ob->ob_type; +} + // bpo-39573: The Py_SET_SIZE() function must be used to set an object size. -static inline Py_ssize_t Py_SIZE(PyObject *ob) { +static inline Py_ssize_t +_Py_SIZE_impl(PyObject *ob) +{ assert(Py_TYPE(ob) != &PyLong_Type); assert(Py_TYPE(ob) != &PyBool_Type); return _PyVarObject_CAST(ob)->ob_size; } -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SIZE(ob) Py_SIZE(_PyObject_CAST(ob)) -#endif -#endif // !defined(_Py_OPAQUE_PYOBJECT) -static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { +static inline int +_Py_IS_TYPE_impl(PyObject *ob, PyTypeObject *type) +{ return Py_TYPE(ob) == type; } -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_IS_TYPE(ob, type) Py_IS_TYPE(_PyObject_CAST(ob), (type)) -#endif - -#ifndef _Py_OPAQUE_PYOBJECT -static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { - ob->ob_type = type; -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SET_TYPE(ob, type) Py_SET_TYPE(_PyObject_CAST(ob), type) -#endif - -static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { +static inline void +_Py_SET_SIZE_impl(PyVarObject *ob, Py_ssize_t size) +{ assert(Py_TYPE(_PyObject_CAST(ob)) != &PyLong_Type); assert(Py_TYPE(_PyObject_CAST(ob)) != &PyBool_Type); #ifdef Py_GIL_DISABLED @@ -321,9 +260,7 @@ static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { ob->ob_size = size; #endif } -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SET_SIZE(ob, size) Py_SET_SIZE(_PyVarObject_CAST(ob), (size)) -#endif + #endif // !defined(_Py_OPAQUE_PYOBJECT) @@ -369,6 +306,11 @@ typedef Py_hash_t (*hashfunc)(PyObject *); typedef PyObject *(*richcmpfunc) (PyObject *, PyObject *, int); typedef PyObject *(*getiterfunc) (PyObject *); typedef PyObject *(*iternextfunc) (PyObject *); +typedef struct { + PyObject *object; + Py_ssize_t index; +} _PyObjectIndexPair; +typedef _PyObjectIndexPair (*_Py_iteritemfunc) (PyObject *, Py_ssize_t index); typedef PyObject *(*descrgetfunc) (PyObject *, PyObject *, PyObject *); typedef int (*descrsetfunc) (PyObject *, PyObject *, PyObject *); typedef int (*initproc)(PyObject *, PyObject *, PyObject *); @@ -527,7 +469,7 @@ given type object has a specified feature. #define Py_TPFLAGS_INLINE_VALUES (1 << 2) /* Placement of weakref pointers are managed by the VM, not by the type. - * The VM will automatically set tp_weaklistoffset. + * The VM will automatically set tp_weaklistoffset. Implies Py_TPFLAGS_HAVE_GC. */ #define Py_TPFLAGS_MANAGED_WEAKREF (1 << 3) @@ -634,9 +576,12 @@ given type object has a specified feature. // Flag values for ob_flags (16 bits available, if SIZEOF_VOID_P > 4). #define _Py_IMMORTAL_FLAGS (1 << 0) +#define _Py_LEGACY_ABI_CHECK_FLAG (1 << 1) /* see PyModuleDef_Init() */ #define _Py_STATICALLY_ALLOCATED_FLAG (1 << 2) -#if defined(Py_GIL_DISABLED) && defined(Py_DEBUG) -#define _Py_TYPE_REVEALED_FLAG (1 << 3) +#if !defined(Py_LIMITED_API) +# if defined(Py_GIL_DISABLED) && defined(Py_DEBUG) +# define _Py_TYPE_REVEALED_FLAG (1 << 3) +# endif #endif #define Py_CONSTANT_NONE 0 @@ -692,8 +637,13 @@ PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */ # define Py_NotImplemented (&_Py_NotImplementedStruct) #endif -/* Macro for returning Py_NotImplemented from a function */ -#define Py_RETURN_NOTIMPLEMENTED return Py_NotImplemented +/* Macro for returning Py_NotImplemented from a function. Only treat + * Py_NotImplemented as immortal in the limited C API 3.12 and newer. */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030c0000 +# define Py_RETURN_NOTIMPLEMENTED return Py_NewRef(Py_NotImplemented) +#else +# define Py_RETURN_NOTIMPLEMENTED return Py_NotImplemented +#endif /* Rich comparison opcodes */ #define Py_LT 0 @@ -831,6 +781,19 @@ PyAPI_FUNC(PyObject *) PyType_GetModuleByDef(PyTypeObject *, PyModuleDef *); PyAPI_FUNC(int) PyType_Freeze(PyTypeObject *type); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(PyObject *) PyType_GetModuleByToken(PyTypeObject *type, + const void *token); +PyAPI_FUNC(void *) PyObject_GetTypeData_DuringGC(PyObject *obj, + PyTypeObject *cls); +PyAPI_FUNC(void *) PyType_GetModuleState_DuringGC(PyTypeObject *); +PyAPI_FUNC(int) PyType_GetBaseByToken_DuringGC(PyTypeObject *, + void *, PyTypeObject **); +PyAPI_FUNC(PyObject *) PyType_GetModule_DuringGC(PyTypeObject *); +PyAPI_FUNC(PyObject *) PyType_GetModuleByToken_DuringGC(PyTypeObject *type, + const void *token); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 1d5c74adefc..53a7eaf0de5 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -26,111 +26,110 @@ extern "C" { #define FORMAT_WITH_SPEC 13 #define GET_AITER 14 #define GET_ANEXT 15 -#define GET_ITER 16 +#define GET_LEN 16 #define RESERVED 17 -#define GET_LEN 18 -#define GET_YIELD_FROM_ITER 19 -#define INTERPRETER_EXIT 20 -#define LOAD_BUILD_CLASS 21 -#define LOAD_LOCALS 22 -#define MAKE_FUNCTION 23 -#define MATCH_KEYS 24 -#define MATCH_MAPPING 25 -#define MATCH_SEQUENCE 26 -#define NOP 27 -#define NOT_TAKEN 28 -#define POP_EXCEPT 29 -#define POP_ITER 30 -#define POP_TOP 31 -#define PUSH_EXC_INFO 32 -#define PUSH_NULL 33 -#define RETURN_GENERATOR 34 -#define RETURN_VALUE 35 -#define SETUP_ANNOTATIONS 36 -#define STORE_SLICE 37 -#define STORE_SUBSCR 38 -#define TO_BOOL 39 -#define UNARY_INVERT 40 -#define UNARY_NEGATIVE 41 -#define UNARY_NOT 42 -#define WITH_EXCEPT_START 43 -#define BINARY_OP 44 -#define BUILD_INTERPOLATION 45 -#define BUILD_LIST 46 -#define BUILD_MAP 47 -#define BUILD_SET 48 -#define BUILD_SLICE 49 -#define BUILD_STRING 50 -#define BUILD_TUPLE 51 -#define CALL 52 -#define CALL_INTRINSIC_1 53 -#define CALL_INTRINSIC_2 54 -#define CALL_KW 55 -#define COMPARE_OP 56 -#define CONTAINS_OP 57 -#define CONVERT_VALUE 58 -#define COPY 59 -#define COPY_FREE_VARS 60 -#define DELETE_ATTR 61 -#define DELETE_DEREF 62 -#define DELETE_FAST 63 -#define DELETE_GLOBAL 64 -#define DELETE_NAME 65 -#define DICT_MERGE 66 -#define DICT_UPDATE 67 -#define END_ASYNC_FOR 68 -#define EXTENDED_ARG 69 -#define FOR_ITER 70 -#define GET_AWAITABLE 71 -#define IMPORT_FROM 72 -#define IMPORT_NAME 73 -#define IS_OP 74 -#define JUMP_BACKWARD 75 -#define JUMP_BACKWARD_NO_INTERRUPT 76 -#define JUMP_FORWARD 77 -#define LIST_APPEND 78 -#define LIST_EXTEND 79 -#define LOAD_ATTR 80 -#define LOAD_COMMON_CONSTANT 81 -#define LOAD_CONST 82 -#define LOAD_DEREF 83 -#define LOAD_FAST 84 -#define LOAD_FAST_AND_CLEAR 85 -#define LOAD_FAST_BORROW 86 -#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 87 -#define LOAD_FAST_CHECK 88 -#define LOAD_FAST_LOAD_FAST 89 -#define LOAD_FROM_DICT_OR_DEREF 90 -#define LOAD_FROM_DICT_OR_GLOBALS 91 -#define LOAD_GLOBAL 92 -#define LOAD_NAME 93 -#define LOAD_SMALL_INT 94 -#define LOAD_SPECIAL 95 -#define LOAD_SUPER_ATTR 96 -#define MAKE_CELL 97 -#define MAP_ADD 98 -#define MATCH_CLASS 99 -#define POP_JUMP_IF_FALSE 100 -#define POP_JUMP_IF_NONE 101 -#define POP_JUMP_IF_NOT_NONE 102 -#define POP_JUMP_IF_TRUE 103 -#define RAISE_VARARGS 104 -#define RERAISE 105 -#define SEND 106 -#define SET_ADD 107 -#define SET_FUNCTION_ATTRIBUTE 108 -#define SET_UPDATE 109 -#define STORE_ATTR 110 -#define STORE_DEREF 111 -#define STORE_FAST 112 -#define STORE_FAST_LOAD_FAST 113 -#define STORE_FAST_STORE_FAST 114 -#define STORE_GLOBAL 115 -#define STORE_NAME 116 -#define SWAP 117 -#define UNPACK_EX 118 -#define UNPACK_SEQUENCE 119 -#define YIELD_VALUE 120 +#define INTERPRETER_EXIT 18 +#define LOAD_BUILD_CLASS 19 +#define LOAD_LOCALS 20 +#define MAKE_FUNCTION 21 +#define MATCH_KEYS 22 +#define MATCH_MAPPING 23 +#define MATCH_SEQUENCE 24 +#define NOP 25 +#define NOT_TAKEN 26 +#define POP_EXCEPT 27 +#define POP_ITER 28 +#define POP_TOP 29 +#define PUSH_EXC_INFO 30 +#define PUSH_NULL 31 +#define RETURN_GENERATOR 32 +#define RETURN_VALUE 33 +#define SETUP_ANNOTATIONS 34 +#define STORE_SLICE 35 +#define STORE_SUBSCR 36 +#define TO_BOOL 37 +#define UNARY_INVERT 38 +#define UNARY_NEGATIVE 39 +#define UNARY_NOT 40 +#define WITH_EXCEPT_START 41 +#define BINARY_OP 42 +#define BUILD_INTERPOLATION 43 +#define BUILD_LIST 44 +#define BUILD_MAP 45 +#define BUILD_SET 46 +#define BUILD_SLICE 47 +#define BUILD_STRING 48 +#define BUILD_TUPLE 49 +#define CALL 50 +#define CALL_INTRINSIC_1 51 +#define CALL_INTRINSIC_2 52 +#define CALL_KW 53 +#define COMPARE_OP 54 +#define CONTAINS_OP 55 +#define CONVERT_VALUE 56 +#define COPY 57 +#define COPY_FREE_VARS 58 +#define DELETE_ATTR 59 +#define DELETE_DEREF 60 +#define DELETE_FAST 61 +#define DELETE_GLOBAL 62 +#define DELETE_NAME 63 +#define DICT_MERGE 64 +#define DICT_UPDATE 65 +#define END_ASYNC_FOR 66 +#define EXTENDED_ARG 67 +#define FOR_ITER 68 +#define GET_AWAITABLE 69 +#define GET_ITER 70 +#define IMPORT_FROM 71 +#define IMPORT_NAME 72 +#define IS_OP 73 +#define JUMP_BACKWARD 74 +#define JUMP_BACKWARD_NO_INTERRUPT 75 +#define JUMP_FORWARD 76 +#define LIST_APPEND 77 +#define LIST_EXTEND 78 +#define LOAD_ATTR 79 +#define LOAD_COMMON_CONSTANT 80 +#define LOAD_CONST 81 +#define LOAD_DEREF 82 +#define LOAD_FAST 83 +#define LOAD_FAST_AND_CLEAR 84 +#define LOAD_FAST_BORROW 85 +#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 86 +#define LOAD_FAST_CHECK 87 +#define LOAD_FAST_LOAD_FAST 88 +#define LOAD_FROM_DICT_OR_DEREF 89 +#define LOAD_FROM_DICT_OR_GLOBALS 90 +#define LOAD_GLOBAL 91 +#define LOAD_NAME 92 +#define LOAD_SMALL_INT 93 +#define LOAD_SPECIAL 94 +#define LOAD_SUPER_ATTR 95 +#define MAKE_CELL 96 +#define MAP_ADD 97 +#define MATCH_CLASS 98 +#define POP_JUMP_IF_FALSE 99 +#define POP_JUMP_IF_NONE 100 +#define POP_JUMP_IF_NOT_NONE 101 +#define POP_JUMP_IF_TRUE 102 +#define RAISE_VARARGS 103 +#define RERAISE 104 +#define SEND 105 +#define SET_ADD 106 +#define SET_FUNCTION_ATTRIBUTE 107 +#define SET_UPDATE 108 +#define STORE_ATTR 109 +#define STORE_DEREF 110 +#define STORE_FAST 111 +#define STORE_FAST_LOAD_FAST 112 +#define STORE_FAST_STORE_FAST 113 +#define STORE_GLOBAL 114 +#define STORE_NAME 115 +#define SWAP 116 +#define UNPACK_EX 117 +#define UNPACK_SEQUENCE 118 +#define YIELD_VALUE 119 #define RESUME 128 #define BINARY_OP_ADD_FLOAT 129 #define BINARY_OP_ADD_INT 130 @@ -144,97 +143,105 @@ extern "C" { #define BINARY_OP_SUBSCR_LIST_SLICE 138 #define BINARY_OP_SUBSCR_STR_INT 139 #define BINARY_OP_SUBSCR_TUPLE_INT 140 -#define BINARY_OP_SUBTRACT_FLOAT 141 -#define BINARY_OP_SUBTRACT_INT 142 -#define CALL_ALLOC_AND_ENTER_INIT 143 -#define CALL_BOUND_METHOD_EXACT_ARGS 144 -#define CALL_BOUND_METHOD_GENERAL 145 -#define CALL_BUILTIN_CLASS 146 -#define CALL_BUILTIN_FAST 147 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 148 -#define CALL_BUILTIN_O 149 -#define CALL_ISINSTANCE 150 -#define CALL_KW_BOUND_METHOD 151 -#define CALL_KW_NON_PY 152 -#define CALL_KW_PY 153 -#define CALL_LEN 154 -#define CALL_LIST_APPEND 155 -#define CALL_METHOD_DESCRIPTOR_FAST 156 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 157 -#define CALL_METHOD_DESCRIPTOR_NOARGS 158 -#define CALL_METHOD_DESCRIPTOR_O 159 -#define CALL_NON_PY_GENERAL 160 -#define CALL_PY_EXACT_ARGS 161 -#define CALL_PY_GENERAL 162 -#define CALL_STR_1 163 -#define CALL_TUPLE_1 164 -#define CALL_TYPE_1 165 -#define COMPARE_OP_FLOAT 166 -#define COMPARE_OP_INT 167 -#define COMPARE_OP_STR 168 -#define CONTAINS_OP_DICT 169 -#define CONTAINS_OP_SET 170 -#define FOR_ITER_GEN 171 -#define FOR_ITER_LIST 172 -#define FOR_ITER_RANGE 173 -#define FOR_ITER_TUPLE 174 -#define JUMP_BACKWARD_JIT 175 -#define JUMP_BACKWARD_NO_JIT 176 -#define LOAD_ATTR_CLASS 177 -#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 178 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 179 -#define LOAD_ATTR_INSTANCE_VALUE 180 -#define LOAD_ATTR_METHOD_LAZY_DICT 181 -#define LOAD_ATTR_METHOD_NO_DICT 182 -#define LOAD_ATTR_METHOD_WITH_VALUES 183 -#define LOAD_ATTR_MODULE 184 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 185 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 186 -#define LOAD_ATTR_PROPERTY 187 -#define LOAD_ATTR_SLOT 188 -#define LOAD_ATTR_WITH_HINT 189 -#define LOAD_GLOBAL_BUILTIN 190 -#define LOAD_GLOBAL_MODULE 191 -#define LOAD_SUPER_ATTR_ATTR 192 -#define LOAD_SUPER_ATTR_METHOD 193 -#define RESUME_CHECK 194 -#define SEND_GEN 195 -#define STORE_ATTR_INSTANCE_VALUE 196 -#define STORE_ATTR_SLOT 197 -#define STORE_ATTR_WITH_HINT 198 -#define STORE_SUBSCR_DICT 199 -#define STORE_SUBSCR_LIST_INT 200 -#define TO_BOOL_ALWAYS_TRUE 201 -#define TO_BOOL_BOOL 202 -#define TO_BOOL_INT 203 -#define TO_BOOL_LIST 204 -#define TO_BOOL_NONE 205 -#define TO_BOOL_STR 206 -#define UNPACK_SEQUENCE_LIST 207 -#define UNPACK_SEQUENCE_TUPLE 208 -#define UNPACK_SEQUENCE_TWO_TUPLE 209 -#define INSTRUMENTED_END_FOR 234 -#define INSTRUMENTED_POP_ITER 235 -#define INSTRUMENTED_END_SEND 236 -#define INSTRUMENTED_FOR_ITER 237 -#define INSTRUMENTED_INSTRUCTION 238 -#define INSTRUMENTED_JUMP_FORWARD 239 -#define INSTRUMENTED_NOT_TAKEN 240 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 241 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 242 -#define INSTRUMENTED_POP_JUMP_IF_NONE 243 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 244 -#define INSTRUMENTED_RESUME 245 -#define INSTRUMENTED_RETURN_VALUE 246 -#define INSTRUMENTED_YIELD_VALUE 247 -#define INSTRUMENTED_END_ASYNC_FOR 248 -#define INSTRUMENTED_LOAD_SUPER_ATTR 249 -#define INSTRUMENTED_CALL 250 -#define INSTRUMENTED_CALL_KW 251 -#define INSTRUMENTED_CALL_FUNCTION_EX 252 -#define INSTRUMENTED_JUMP_BACKWARD 253 -#define INSTRUMENTED_LINE 254 -#define ENTER_EXECUTOR 255 +#define BINARY_OP_SUBSCR_USTR_INT 141 +#define BINARY_OP_SUBTRACT_FLOAT 142 +#define BINARY_OP_SUBTRACT_INT 143 +#define CALL_ALLOC_AND_ENTER_INIT 144 +#define CALL_BOUND_METHOD_EXACT_ARGS 145 +#define CALL_BOUND_METHOD_GENERAL 146 +#define CALL_BUILTIN_CLASS 147 +#define CALL_BUILTIN_FAST 148 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 149 +#define CALL_BUILTIN_O 150 +#define CALL_EX_NON_PY_GENERAL 151 +#define CALL_EX_PY 152 +#define CALL_ISINSTANCE 153 +#define CALL_KW_BOUND_METHOD 154 +#define CALL_KW_NON_PY 155 +#define CALL_KW_PY 156 +#define CALL_LEN 157 +#define CALL_LIST_APPEND 158 +#define CALL_METHOD_DESCRIPTOR_FAST 159 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 160 +#define CALL_METHOD_DESCRIPTOR_NOARGS 161 +#define CALL_METHOD_DESCRIPTOR_O 162 +#define CALL_NON_PY_GENERAL 163 +#define CALL_PY_EXACT_ARGS 164 +#define CALL_PY_GENERAL 165 +#define CALL_STR_1 166 +#define CALL_TUPLE_1 167 +#define CALL_TYPE_1 168 +#define COMPARE_OP_FLOAT 169 +#define COMPARE_OP_INT 170 +#define COMPARE_OP_STR 171 +#define CONTAINS_OP_DICT 172 +#define CONTAINS_OP_SET 173 +#define FOR_ITER_GEN 174 +#define FOR_ITER_LIST 175 +#define FOR_ITER_RANGE 176 +#define FOR_ITER_TUPLE 177 +#define FOR_ITER_VIRTUAL 178 +#define GET_ITER_SELF 179 +#define GET_ITER_VIRTUAL 180 +#define JUMP_BACKWARD_JIT 181 +#define JUMP_BACKWARD_NO_JIT 182 +#define LOAD_ATTR_CLASS 183 +#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 184 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 185 +#define LOAD_ATTR_INSTANCE_VALUE 186 +#define LOAD_ATTR_METHOD_LAZY_DICT 187 +#define LOAD_ATTR_METHOD_NO_DICT 188 +#define LOAD_ATTR_METHOD_WITH_VALUES 189 +#define LOAD_ATTR_MODULE 190 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 191 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 192 +#define LOAD_ATTR_PROPERTY 193 +#define LOAD_ATTR_SLOT 194 +#define LOAD_ATTR_WITH_HINT 195 +#define LOAD_GLOBAL_BUILTIN 196 +#define LOAD_GLOBAL_MODULE 197 +#define LOAD_SUPER_ATTR_ATTR 198 +#define LOAD_SUPER_ATTR_METHOD 199 +#define RESUME_CHECK 200 +#define RESUME_CHECK_JIT 201 +#define SEND_GEN 202 +#define STORE_ATTR_INSTANCE_VALUE 203 +#define STORE_ATTR_SLOT 204 +#define STORE_ATTR_WITH_HINT 205 +#define STORE_SUBSCR_DICT 206 +#define STORE_SUBSCR_LIST_INT 207 +#define TO_BOOL_ALWAYS_TRUE 208 +#define TO_BOOL_BOOL 209 +#define TO_BOOL_INT 210 +#define TO_BOOL_LIST 211 +#define TO_BOOL_NONE 212 +#define TO_BOOL_STR 213 +#define UNPACK_SEQUENCE_LIST 214 +#define UNPACK_SEQUENCE_TUPLE 215 +#define UNPACK_SEQUENCE_TWO_TUPLE 216 +#define INSTRUMENTED_END_FOR 233 +#define INSTRUMENTED_POP_ITER 234 +#define INSTRUMENTED_END_SEND 235 +#define INSTRUMENTED_FOR_ITER 236 +#define INSTRUMENTED_INSTRUCTION 237 +#define INSTRUMENTED_JUMP_FORWARD 238 +#define INSTRUMENTED_NOT_TAKEN 239 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 240 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 241 +#define INSTRUMENTED_POP_JUMP_IF_NONE 242 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 243 +#define INSTRUMENTED_RESUME 244 +#define INSTRUMENTED_RETURN_VALUE 245 +#define INSTRUMENTED_YIELD_VALUE 246 +#define INSTRUMENTED_END_ASYNC_FOR 247 +#define INSTRUMENTED_LOAD_SUPER_ATTR 248 +#define INSTRUMENTED_CALL 249 +#define INSTRUMENTED_CALL_KW 250 +#define INSTRUMENTED_CALL_FUNCTION_EX 251 +#define INSTRUMENTED_JUMP_BACKWARD 252 +#define INSTRUMENTED_LINE 253 +#define ENTER_EXECUTOR 254 +#define TRACE_RECORD 255 #define ANNOTATIONS_PLACEHOLDER 256 #define JUMP 257 #define JUMP_IF_FALSE 258 @@ -247,9 +254,9 @@ extern "C" { #define SETUP_WITH 265 #define STORE_FAST_MAYBE_NULL 266 -#define HAVE_ARGUMENT 43 +#define HAVE_ARGUMENT 41 #define MIN_SPECIALIZED_OPCODE 129 -#define MIN_INSTRUMENTED_OPCODE 234 +#define MIN_INSTRUMENTED_OPCODE 233 #ifdef __cplusplus } diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 63ebbb32d06..974246f896e 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -24,10 +24,10 @@ #define PY_MINOR_VERSION 15 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 0 +#define PY_RELEASE_SERIAL 8 /* Version as a string */ -#define PY_VERSION "3.15.0a0" +#define PY_VERSION "3.15.0a8+" /*--end constants--*/ diff --git a/Include/pyabi.h b/Include/pyabi.h new file mode 100644 index 00000000000..8c4ae281a43 --- /dev/null +++ b/Include/pyabi.h @@ -0,0 +1,121 @@ +/* Macros that restrict available definitions and select implementations + * to match an ABI stability promise: + * + * - internal API/ABI (may change at any time) -- Py_BUILD_CORE* + * - general CPython API/ABI (may change in 3.x.0) -- default + * - Stable ABI: abi3, abi3t (long-term stable) -- Py_LIMITED_API, + * Py_TARGET_ABI3T, _Py_OPAQUE_PYOBJECT + * - Free-threading (incompatible with non-free-threading builds) + * -- Py_GIL_DISABLED + */ + +#ifndef _Py_PYABI_H +#define _Py_PYABI_H + +/* Defines to build Python and its standard library: + * + * - Py_BUILD_CORE: Build Python core. Gives access to Python internals; should + * not be used by third-party modules. + * - Py_BUILD_CORE_BUILTIN: Build a Python stdlib module as a built-in module. + * - Py_BUILD_CORE_MODULE: Build a Python stdlib module as a dynamic library. + * + * Py_BUILD_CORE_BUILTIN and Py_BUILD_CORE_MODULE imply Py_BUILD_CORE. + * + * On Windows, Py_BUILD_CORE_MODULE exports "PyInit_xxx" symbol, whereas + * Py_BUILD_CORE_BUILTIN does not. + */ +#if defined(Py_BUILD_CORE_BUILTIN) && !defined(Py_BUILD_CORE) +# define Py_BUILD_CORE +#endif +#if defined(Py_BUILD_CORE_MODULE) && !defined(Py_BUILD_CORE) +# define Py_BUILD_CORE +#endif + +/* Check valid values for target ABI macros. + */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 3 + // Empty Py_LIMITED_API used to work; redefine to + // Python 3.2 to be explicit. +# undef Py_LIMITED_API +# define Py_LIMITED_API 0x03020000 +#endif +#if defined(Py_TARGET_ABI3T) && Py_TARGET_ABI3T+0 < 0x030f0000 +# error "Py_TARGET_ABI3T must be 0x030f0000 (3.15) or above" +#endif + +/* Stable ABI for free-threaded builds (abi3t, introduced in PEP 803) + * is enabled by one of: + * - Py_TARGET_ABI3T, or + * - Py_LIMITED_API and Py_GIL_DISABLED. + * + * These affect set the following, which Python.h should use internally: + * - Py_LIMITED_API (defines the subset of API we expose) + * - _Py_OPAQUE_PYOBJECT (additionally hides what's ABI-incompatible between + * free-threaded & GIL) + * + * (Don't use Py_TARGET_ABI3T directly. It's currently only used to set these + * 2 macros, and defined for users' convenience.) + */ +#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) \ + && !defined(Py_TARGET_ABI3T) +# define Py_TARGET_ABI3T Py_LIMITED_API +#endif +#if defined(Py_TARGET_ABI3T) +# define _Py_OPAQUE_PYOBJECT +# if !defined(Py_LIMITED_API) +# define Py_LIMITED_API Py_TARGET_ABI3T +# elif Py_LIMITED_API > Py_TARGET_ABI3T + // if both are defined, use the *lower* version, + // i.e. maximum compatibility +# undef Py_LIMITED_API +# define Py_LIMITED_API Py_TARGET_ABI3T +# endif +#else +# ifdef _Py_OPAQUE_PYOBJECT + // _Py_OPAQUE_PYOBJECT is a private macro; do not define it directly. +# error "Define Py_TARGET_ABI3T to target abi3t." +# endif +#endif + +#if defined(Py_TARGET_ABI3T) +# if !defined(Py_GIL_DISABLED) + // Define Py_GIL_DISABLED for users' needs. Users check this macro to see + // whether they need extra synchronization. +# define Py_GIL_DISABLED +# endif +# if defined(_Py_IS_TESTCEXT) + // When compiling for abi3t, contents of Python.h should not depend + // on Py_GIL_DISABLED. + // We ask GCC to error if it sees the macro from this point on. + // Since users are free to the macro, and there's no way to undo the + // poisoning at the end of Python.h, we only do this in a test module + // (test_cext). + // + // Clang's poisoning is stricter than GCC's: it looks in `#elif` + // expressions after matching `#if`s. We disable it for now. + // We also provide an undocumented, unsupported opt-out macro to help + // porting to other compilers. Consider reaching out if you use it. +# if defined(__GNUC__) && !defined(__clang__) && !defined(_Py_NO_GCC_POISON) +# undef Py_GIL_DISABLED +# pragma GCC poison Py_GIL_DISABLED +# endif +# endif +#endif + +/* The internal C API must not be used with the limited C API: make sure + * that Py_BUILD_CORE* macros are not defined in this case. + * But, keep the "original" values, under different names, for "exports.h" + */ +#ifdef Py_BUILD_CORE +# define _PyEXPORTS_CORE +#endif +#ifdef Py_BUILD_CORE_MODULE +# define _PyEXPORTS_CORE_MODULE +#endif +#ifdef Py_LIMITED_API +# undef Py_BUILD_CORE +# undef Py_BUILD_CORE_BUILTIN +# undef Py_BUILD_CORE_MODULE +#endif + +#endif // _Py_PYABI_H diff --git a/Include/pybuffer.h b/Include/pybuffer.h index ca1c6058d90..6371b9b4837 100644 --- a/Include/pybuffer.h +++ b/Include/pybuffer.h @@ -67,27 +67,27 @@ PyAPI_FUNC(int) PyBuffer_FromContiguous(const Py_buffer *view, const void *buf, error (i.e. the object does not have a buffer interface or it is not working). - If fort is 'F', then if the object is multi-dimensional, + If order is 'F', then if the object is multi-dimensional, then the data will be copied into the array in Fortran-style (first dimension varies the fastest). If - fort is 'C', then the data will be copied into the array - in C-style (last dimension varies the fastest). If fort + order is 'C', then the data will be copied into the array + in C-style (last dimension varies the fastest). If order is 'A', then it does not matter and the copy will be made in whatever way is more efficient. */ PyAPI_FUNC(int) PyObject_CopyData(PyObject *dest, PyObject *src); /* Copy the data from the src buffer to the buffer of destination. */ -PyAPI_FUNC(int) PyBuffer_IsContiguous(const Py_buffer *view, char fort); +PyAPI_FUNC(int) PyBuffer_IsContiguous(const Py_buffer *view, char order); /*Fill the strides array with byte-strides of a contiguous - (Fortran-style if fort is 'F' or C-style otherwise) + (Fortran-style if order is 'F' or C-style otherwise) array of the given shape with the given number of bytes per element. */ PyAPI_FUNC(void) PyBuffer_FillContiguousStrides(int ndims, Py_ssize_t *shape, Py_ssize_t *strides, int itemsize, - char fort); + char order); /* Fills in a buffer-info structure correctly for an exporter that can only share a contiguous chunk of memory of diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 5d0028c116e..cfabbc5fe8d 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -91,6 +91,9 @@ PyAPI_DATA(PyObject *) PyExc_EOFError; PyAPI_DATA(PyObject *) PyExc_FloatingPointError; PyAPI_DATA(PyObject *) PyExc_OSError; PyAPI_DATA(PyObject *) PyExc_ImportError; +#if !defined(Py_LIMITED_API) +PyAPI_DATA(PyObject *) PyExc_ImportCycleError; +#endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03060000 PyAPI_DATA(PyObject *) PyExc_ModuleNotFoundError; #endif diff --git a/Include/pyexpat.h b/Include/pyexpat.h index 9824d099c3d..f523f8bb273 100644 --- a/Include/pyexpat.h +++ b/Include/pyexpat.h @@ -52,6 +52,16 @@ struct PyExpat_CAPI int (*SetHashSalt)(XML_Parser parser, unsigned long hash_salt); /* might be NULL for expat < 2.6.0 */ XML_Bool (*SetReparseDeferralEnabled)(XML_Parser parser, XML_Bool enabled); + /* might be NULL for expat < 2.7.2 */ + XML_Bool (*SetAllocTrackerActivationThreshold)( + XML_Parser parser, unsigned long long activationThresholdBytes); + XML_Bool (*SetAllocTrackerMaximumAmplification)( + XML_Parser parser, float maxAmplificationFactor); + /* might be NULL for expat < 2.4.0 */ + XML_Bool (*SetBillionLaughsAttackProtectionActivationThreshold)( + XML_Parser parser, unsigned long long activationThresholdBytes); + XML_Bool (*SetBillionLaughsAttackProtectionMaximumAmplification)( + XML_Parser parser, float maxAmplificationFactor); /* always add new stuff to the end! */ }; diff --git a/Include/pylock.h b/Include/pylock.h deleted file mode 100644 index 1939ef269d3..00000000000 --- a/Include/pylock.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef Py_LOCK_H -#define Py_LOCK_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_LOCK_H -# include "cpython/pylock.h" -# undef Py_CPYTHON_LOCK_H -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_LOCK_H */ diff --git a/Include/pymacconfig.h b/Include/pymacconfig.h index 615abe103ca..9d63ddf8a71 100644 --- a/Include/pymacconfig.h +++ b/Include/pymacconfig.h @@ -18,7 +18,6 @@ #undef SIZEOF_UINTPTR_T #undef SIZEOF_PTHREAD_T #undef WORDS_BIGENDIAN -#undef DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754 #undef DOUBLE_IS_BIG_ENDIAN_IEEE754 #undef DOUBLE_IS_LITTLE_ENDIAN_IEEE754 #undef HAVE_GCC_ASM_FOR_X87 diff --git a/Include/pymacro.h b/Include/pymacro.h index 857cdf12db9..7ecce44a0d2 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -116,6 +116,12 @@ /* Absolute value of the number x */ #define Py_ABS(x) ((x) < 0 ? -(x) : (x)) +/* Safer implementation that avoids an undefined behavior for the minimal + value of the signed integer type if its absolute value is larger than + the maximal value of the signed integer type (in the two's complement + representations, which is common). + */ +#define _Py_ABS_CAST(T, x) ((x) >= 0 ? ((T) (x)) : ((T) (((T) -((x) + 1)) + 1u))) #define _Py_XSTRINGIFY(x) #x diff --git a/Include/pymath.h b/Include/pymath.h index e2919c7b527..7cfe441365d 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -7,6 +7,7 @@ /* High precision definition of pi and e (Euler) * The values are taken from libc6's math.h. */ +// Deprecated since Python 3.15. #ifndef Py_MATH_PIl #define Py_MATH_PIl 3.1415926535897932384626433832795029L #endif @@ -14,6 +15,7 @@ #define Py_MATH_PI 3.14159265358979323846 #endif +// Deprecated since Python 3.15. #ifndef Py_MATH_El #define Py_MATH_El 2.7182818284590452353602874713526625L #endif @@ -43,13 +45,14 @@ #define Py_IS_FINITE(X) isfinite(X) // Py_INFINITY: Value that evaluates to a positive double infinity. +// Soft deprecated since Python 3.15, use INFINITY instead. #ifndef Py_INFINITY # define Py_INFINITY ((double)INFINITY) #endif /* Py_HUGE_VAL should always be the same as Py_INFINITY. But historically * this was not reliable and Python did not require IEEE floats and C99 - * conformity. The macro was soft deprecated in Python 3.14, use Py_INFINITY instead. + * conformity. The macro was soft deprecated in Python 3.14, use INFINITY instead. */ #ifndef Py_HUGE_VAL # define Py_HUGE_VAL HUGE_VAL diff --git a/Include/pyport.h b/Include/pyport.h index 62db8d07701..73a3e6cdaf0 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -58,26 +58,6 @@ #endif -/* Defines to build Python and its standard library: - * - * - Py_BUILD_CORE: Build Python core. Give access to Python internals, but - * should not be used by third-party modules. - * - Py_BUILD_CORE_BUILTIN: Build a Python stdlib module as a built-in module. - * - Py_BUILD_CORE_MODULE: Build a Python stdlib module as a dynamic library. - * - * Py_BUILD_CORE_BUILTIN and Py_BUILD_CORE_MODULE imply Py_BUILD_CORE. - * - * On Windows, Py_BUILD_CORE_MODULE exports "PyInit_xxx" symbol, whereas - * Py_BUILD_CORE_BUILTIN does not. - */ -#if defined(Py_BUILD_CORE_BUILTIN) && !defined(Py_BUILD_CORE) -# define Py_BUILD_CORE -#endif -#if defined(Py_BUILD_CORE_MODULE) && !defined(Py_BUILD_CORE) -# define Py_BUILD_CORE -#endif - - /************************************************************************** Symbols and macros to supply platform-independent interfaces to basic C language & library operations whose spellings vary across platforms. @@ -385,17 +365,6 @@ extern "C" { # define Py_NO_INLINE #endif -#include "exports.h" - -#ifdef Py_LIMITED_API - // The internal C API must not be used with the limited C API: make sure - // that Py_BUILD_CORE macro is not defined in this case. These 3 macros are - // used by exports.h, so only undefine them afterwards. -# undef Py_BUILD_CORE -# undef Py_BUILD_CORE_BUILTIN -# undef Py_BUILD_CORE_MODULE -#endif - /* limits.h constants that may be missing */ #ifndef INT_MAX @@ -446,7 +415,9 @@ extern "C" { /* * Specify alignment on compilers that support it. */ -#if defined(__GNUC__) && __GNUC__ >= 3 +#ifdef Py_BUILD_CORE +// always use _Py_ALIGNED_DEF instead +#elif defined(__GNUC__) && __GNUC__ >= 3 #define Py_ALIGNED(x) __attribute__((aligned(x))) #else #define Py_ALIGNED(x) @@ -504,28 +475,30 @@ extern "C" { * Thread support is stubbed and any attempt to create a new thread fails. */ #if (!defined(HAVE_PTHREAD_STUBS) && \ + !defined(__wasi__) && \ (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))) # define Py_CAN_START_THREADS 1 #endif + +/* gh-142163: Some libraries rely on HAVE_THREAD_LOCAL being undefined, so + * we can only define it only when Py_BUILD_CORE is set.*/ +#ifdef Py_BUILD_CORE +// This is no longer coupled to _Py_thread_local. +# define HAVE_THREAD_LOCAL 1 +#endif + #ifdef WITH_THREAD -# ifdef Py_BUILD_CORE -# ifdef HAVE_THREAD_LOCAL -# error "HAVE_THREAD_LOCAL is already defined" -# endif -# define HAVE_THREAD_LOCAL 1 -# ifdef thread_local -# define _Py_thread_local thread_local -# elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__) -# define _Py_thread_local _Thread_local -# elif defined(_MSC_VER) /* AKA NT_THREADS */ -# define _Py_thread_local __declspec(thread) -# elif defined(__GNUC__) /* includes clang */ -# define _Py_thread_local __thread -# else - // fall back to the PyThread_tss_*() API, or ignore. -# undef HAVE_THREAD_LOCAL -# endif +# ifdef thread_local +# define _Py_thread_local thread_local +# elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__) +# define _Py_thread_local _Thread_local +# elif defined(_MSC_VER) /* AKA NT_THREADS */ +# define _Py_thread_local __declspec(thread) +# elif defined(__GNUC__) /* includes clang */ +# define _Py_thread_local __thread +# else +# error "no supported thread-local variable storage classifier" # endif #endif @@ -565,8 +538,11 @@ extern "C" { // // Example: _Py_TYPEOF(x) x_copy = (x); // -// The macro is only defined if GCC or clang compiler is used. -#if defined(__GNUC__) || defined(__clang__) +// On C23, use typeof(). Otherwise, the macro is only defined +// if GCC or clang compiler is used. +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +# define _Py_TYPEOF(expr) typeof(expr) +#elif defined(__GNUC__) || defined(__clang__) # define _Py_TYPEOF(expr) __typeof__(expr) #endif @@ -577,6 +553,7 @@ extern "C" { # if !defined(_Py_MEMORY_SANITIZER) # define _Py_MEMORY_SANITIZER # define _Py_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory)) +# define _Py_MSAN_UNPOISON(PTR, SIZE) (__msan_unpoison(PTR, SIZE)) # endif # endif # if __has_feature(address_sanitizer) @@ -615,6 +592,9 @@ extern "C" { #ifndef _Py_NO_SANITIZE_MEMORY # define _Py_NO_SANITIZE_MEMORY #endif +#ifndef _Py_MSAN_UNPOISON +# define _Py_MSAN_UNPOISON(PTR, SIZE) +#endif /* AIX has __bool__ redefined in it's system header file. */ #if defined(_AIX) && defined(__bool__) @@ -682,4 +662,10 @@ extern "C" { #endif +// Assume the stack grows down unless specified otherwise +#ifndef _Py_STACK_GROWS_DOWN +# define _Py_STACK_GROWS_DOWN 1 +#endif + + #endif /* Py_PYPORT_H */ diff --git a/Include/refcount.h b/Include/refcount.h index ba34461fefc..80fe7ff70a1 100644 --- a/Include/refcount.h +++ b/Include/refcount.h @@ -5,6 +5,7 @@ extern "C" { #endif +#if !defined(_Py_OPAQUE_PYOBJECT) /* Immortalization: @@ -90,14 +91,16 @@ check by comparing the reference count field to the minimum immortality refcount # define _Py_REF_SHARED(refcnt, flags) \ (((refcnt) << _Py_REF_SHARED_SHIFT) + (flags)) #endif // Py_GIL_DISABLED - +#endif // _Py_OPAQUE_PYOBJECT // Py_REFCNT() implementation for the stable ABI PyAPI_FUNC(Py_ssize_t) Py_REFCNT(PyObject *ob); #if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030e0000 // Stable ABI implements Py_REFCNT() as a function call - // on limited C API version 3.14 and newer. + // on limited C API version 3.14 and newer, and on abi3t. +#elif defined(_Py_OPAQUE_PYOBJECT) + // Py_REFCNT() is also a function call in abi3t. #else static inline Py_ssize_t _Py_REFCNT(PyObject *ob) { #if !defined(Py_GIL_DISABLED) @@ -114,6 +117,8 @@ PyAPI_FUNC(Py_ssize_t) Py_REFCNT(PyObject *ob); } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_REFCNT(ob) _Py_REFCNT(_PyObject_CAST(ob)) + #else + # define Py_REFCNT(ob) _Py_REFCNT(ob) #endif #endif @@ -124,7 +129,7 @@ static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) return (_Py_atomic_load_uint32_relaxed(&op->ob_ref_local) == _Py_IMMORTAL_REFCNT_LOCAL); #elif SIZEOF_VOID_P > 4 - return _Py_CAST(PY_INT32_T, op->ob_refcnt) < 0; + return _Py_CAST(int32_t, op->ob_refcnt) < 0; #else return op->ob_refcnt >= _Py_IMMORTAL_MINIMUM_REFCNT; #endif @@ -148,9 +153,10 @@ PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt); static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { assert(refcnt >= 0); -#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030d0000 +#if (defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030d0000) \ + || defined(_Py_OPAQUE_PYOBJECT) // Stable ABI implements Py_SET_REFCNT() as a function call - // on limited C API version 3.13 and newer. + // on limited C API version 3.13 and newer, and abi3t. _Py_SetRefcnt(ob, refcnt); #else // This immortal check is for code that is unaware of immortal objects. @@ -162,7 +168,7 @@ static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { } #ifndef Py_GIL_DISABLED #if SIZEOF_VOID_P > 4 - ob->ob_refcnt = (PY_UINT32_T)refcnt; + ob->ob_refcnt = (uint32_t)refcnt; #else ob->ob_refcnt = refcnt; #endif @@ -189,7 +195,7 @@ static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { ob->ob_ref_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); } #endif // Py_GIL_DISABLED -#endif // Py_LIMITED_API+0 < 0x030d0000 +#endif // Py_LIMITED_API } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SET_REFCNT(ob, refcnt) Py_SET_REFCNT(_PyObject_CAST(ob), (refcnt)) @@ -248,10 +254,11 @@ PyAPI_FUNC(void) _Py_DecRef(PyObject *); static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) { -#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) +#if (defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))) \ + || defined(_Py_OPAQUE_PYOBJECT) // Stable ABI implements Py_INCREF() as a function call on limited C API - // version 3.12 and newer, and on Python built in debug mode. _Py_IncRef() - // was added to Python 3.10.0a7, use Py_IncRef() on older Python versions. + // version 3.12 and newer, abi3t, and on Python built in debug mode. + // _Py_IncRef() was added to Python 3.10.0a7, use Py_IncRef() on older versions. // Py_IncRef() accepts NULL whereas _Py_IncRef() doesn't. # if Py_LIMITED_API+0 >= 0x030a00A7 _Py_IncRef(op); @@ -276,7 +283,7 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) _Py_atomic_add_ssize(&op->ob_ref_shared, (1 << _Py_REF_SHARED_SHIFT)); } #elif SIZEOF_VOID_P > 4 - PY_UINT32_T cur_refcnt = op->ob_refcnt; + uint32_t cur_refcnt = op->ob_refcnt; if (cur_refcnt >= _Py_IMMORTAL_INITIAL_REFCNT) { // the object is immortal _Py_INCREF_IMMORTAL_STAT_INC(); @@ -303,8 +310,8 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) # define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op)) #endif - -#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) +#if !defined(Py_LIMITED_API) +#if defined(Py_GIL_DISABLED) // Implements Py_DECREF on objects not owned by the current thread. PyAPI_FUNC(void) _Py_DecRefShared(PyObject *); PyAPI_FUNC(void) _Py_DecRefSharedDebug(PyObject *, const char *, int); @@ -314,12 +321,14 @@ PyAPI_FUNC(void) _Py_DecRefSharedDebug(PyObject *, const char *, int); // zero. Otherwise, the thread gives up ownership and merges the reference // count fields. PyAPI_FUNC(void) _Py_MergeZeroLocalRefcount(PyObject *); -#endif +#endif // Py_GIL_DISABLED +#endif // Py_LIMITED_API -#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) +#if (defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))) \ + || defined(_Py_OPAQUE_PYOBJECT) // Stable ABI implements Py_DECREF() as a function call on limited C API -// version 3.12 and newer, and on Python built in debug mode. _Py_DecRef() was -// added to Python 3.10.0a7, use Py_DecRef() on older Python versions. +// version 3.12 and newer, abi3t, and on Python built in debug mode. +// _Py_DecRef() was added to Python 3.10.0a7, use Py_DecRef() on older versions. // Py_DecRef() accepts NULL whereas _Py_DecRef() doesn't. static inline void Py_DECREF(PyObject *op) { # if Py_LIMITED_API+0 >= 0x030a00A7 @@ -385,7 +394,7 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) #if SIZEOF_VOID_P > 4 /* If an object has been freed, it will have a negative full refcnt * If it has not it been freed, will have a very large refcnt */ - if (op->ob_refcnt_full <= 0 || op->ob_refcnt > (((PY_UINT32_T)-1) - (1<<20))) { + if (op->ob_refcnt_full <= 0 || op->ob_refcnt > (((uint32_t)-1) - (1<<20))) { #else if (op->ob_refcnt <= 0) { #endif diff --git a/Include/sliceobject.h b/Include/sliceobject.h index 35e2ea254ca..00c70a6e911 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -16,19 +16,6 @@ PyAPI_DATA(PyObject) _Py_EllipsisObject; /* Don't use this directly */ /* Slice object interface */ -/* - -A slice object containing start, stop, and step data members (the -names are from range). After much talk with Guido, it was decided to -let these be any arbitrary python type. Py_None stands for omitted values. -*/ -#ifndef Py_LIMITED_API -typedef struct { - PyObject_HEAD - PyObject *start, *stop, *step; /* not NULL */ -} PySliceObject; -#endif - PyAPI_DATA(PyTypeObject) PySlice_Type; PyAPI_DATA(PyTypeObject) PyEllipsis_Type; @@ -36,12 +23,6 @@ PyAPI_DATA(PyTypeObject) PyEllipsis_Type; PyAPI_FUNC(PyObject *) PySlice_New(PyObject* start, PyObject* stop, PyObject* step); -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PySlice_FromIndices(Py_ssize_t start, Py_ssize_t stop); -PyAPI_FUNC(int) _PySlice_GetLongIndices(PySliceObject *self, PyObject *length, - PyObject **start_ptr, PyObject **stop_ptr, - PyObject **step_ptr); -#endif PyAPI_FUNC(int) PySlice_GetIndices(PyObject *r, Py_ssize_t length, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step); Py_DEPRECATED(3.7) @@ -63,6 +44,12 @@ PyAPI_FUNC(Py_ssize_t) PySlice_AdjustIndices(Py_ssize_t length, Py_ssize_t step); #endif +#ifndef Py_LIMITED_API +# define Py_CPYTHON_SLICEOBJECT_H +# include "cpython/sliceobject.h" +# undef Py_CPYTHON_SLICEOBJECT_H +#endif + #ifdef __cplusplus } #endif diff --git a/Include/structseq.h b/Include/structseq.h index 29e24fee54e..e52d6188030 100644 --- a/Include/structseq.h +++ b/Include/structseq.h @@ -21,12 +21,6 @@ typedef struct PyStructSequence_Desc { PyAPI_DATA(const char * const) PyStructSequence_UnnamedField; -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) PyStructSequence_InitType(PyTypeObject *type, - PyStructSequence_Desc *desc); -PyAPI_FUNC(int) PyStructSequence_InitType2(PyTypeObject *type, - PyStructSequence_Desc *desc); -#endif PyAPI_FUNC(PyTypeObject*) PyStructSequence_NewType(PyStructSequence_Desc *desc); PyAPI_FUNC(PyObject *) PyStructSequence_New(PyTypeObject* type); @@ -35,9 +29,9 @@ PyAPI_FUNC(void) PyStructSequence_SetItem(PyObject*, Py_ssize_t, PyObject*); PyAPI_FUNC(PyObject*) PyStructSequence_GetItem(PyObject*, Py_ssize_t); #ifndef Py_LIMITED_API -typedef PyTupleObject PyStructSequence; -#define PyStructSequence_SET_ITEM PyStructSequence_SetItem -#define PyStructSequence_GET_ITEM PyStructSequence_GetItem +# define Py_CPYTHON_STRUCTSEQ_H +# include "cpython/structseq.h" +# undef Py_CPYTHON_STRUCTSEQ_H #endif #ifdef __cplusplus diff --git a/Include/sysmodule.h b/Include/sysmodule.h index 2f362791797..b7d800c5e5d 100644 --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -23,8 +23,6 @@ PyAPI_FUNC(void) PySys_WriteStderr(const char *format, ...) PyAPI_FUNC(void) PySys_FormatStdout(const char *format, ...); PyAPI_FUNC(void) PySys_FormatStderr(const char *format, ...); -Py_DEPRECATED(3.13) PyAPI_FUNC(void) PySys_ResetWarnOptions(void); - PyAPI_FUNC(PyObject *) PySys_GetXOptions(void); #ifdef __cplusplus diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index b72d581ec25..29f1d1b01c1 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -64,13 +64,9 @@ Copyright (c) Corporation for National Research Initiatives. #error Must define SIZEOF_WCHAR_T #endif +/* Soft-deprecated defines */ #define Py_UNICODE_SIZE SIZEOF_WCHAR_T - -/* If wchar_t can be used for UCS-4 storage, set Py_UNICODE_WIDE. - Otherwise, Unicode strings are stored as UCS-2 (with limited support - for UTF-16) */ - -#if Py_UNICODE_SIZE >= 4 +#if SIZEOF_WCHAR_T >= 4 #define Py_UNICODE_WIDE #endif diff --git a/Include/weakrefobject.h b/Include/weakrefobject.h index a6e71eb178b..17fac62961c 100644 --- a/Include/weakrefobject.h +++ b/Include/weakrefobject.h @@ -27,7 +27,6 @@ PyAPI_FUNC(PyObject *) PyWeakref_NewRef(PyObject *ob, PyObject *callback); PyAPI_FUNC(PyObject *) PyWeakref_NewProxy(PyObject *ob, PyObject *callback); -Py_DEPRECATED(3.13) PyAPI_FUNC(PyObject *) PyWeakref_GetObject(PyObject *ref); #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030D0000 PyAPI_FUNC(int) PyWeakref_GetRef(PyObject *ref, PyObject **pobj); diff --git a/InternalDocs/README.md b/InternalDocs/README.md index a6e2df5ae4a..3e8ab442315 100644 --- a/InternalDocs/README.md +++ b/InternalDocs/README.md @@ -11,6 +11,11 @@ # CPython Internals Documentation [issue tracker](https://github.com/python/cpython/issues). +General Resources +--- + +- [Source Code Structure](structure.md) + Compiling Python Source Code --- @@ -29,13 +34,13 @@ # CPython Internals Documentation - [Frames](frames.md) -- [String Interning](string_interning.md) - Program Execution --- - [The Bytecode Interpreter](interpreter.md) +- [Stack references (_PyStackRef)](stackrefs.md) + - [The JIT](jit.md) - [Garbage Collector Design](garbage_collector.md) @@ -46,6 +51,14 @@ # CPython Internals Documentation - [Stack protection](stack_protection.md) +Built-in Types +--- + +- [String Interning](string_interning.md) + +- [List sort algorithm](../Objects/listsort.txt) + + Modules --- diff --git a/InternalDocs/asyncio.md b/InternalDocs/asyncio.md index 22159852ca5..ca7e8a92dec 100644 --- a/InternalDocs/asyncio.md +++ b/InternalDocs/asyncio.md @@ -205,7 +205,7 @@ ## Per-thread current task found, `None` is returned. In free-threading, it avoids contention on a global dictionary as -threads can access the current task of thier running loop without any +threads can access the current task of their running loop without any locking. --- diff --git a/InternalDocs/code_objects.md b/InternalDocs/code_objects.md index a91a7043c1b..cccbe715886 100644 --- a/InternalDocs/code_objects.md +++ b/InternalDocs/code_objects.md @@ -70,14 +70,6 @@ ### Format of the locations table representation of the source code positions of instructions, which are returned by the `co_positions()` iterator. -> [!NOTE] -> `co_linetable` is not to be confused with `co_lnotab`. -> For backwards compatibility, `co_lnotab` exposes the format -> as it existed in Python 3.10 and lower: this older format -> stores only the start line for each instruction. -> It is lazily created from `co_linetable` when accessed. -> See [`Objects/lnotab_notes.txt`](../Objects/lnotab_notes.txt) for more details. - `co_linetable` consists of a sequence of location entries. Each entry starts with a byte with the most significant bit set, followed by zero or more bytes with the most significant bit unset. diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md index 02bbdf6071f..742af5efcf5 100644 --- a/InternalDocs/compiler.md +++ b/InternalDocs/compiler.md @@ -604,4 +604,4 @@ pp. 213--227, 1997. [^2]: The Zephyr Abstract Syntax Description Language.: - https://www.cs.princeton.edu/research/techreps/TR-554-97 + https://www.cs.princeton.edu/research/techreps/254 diff --git a/InternalDocs/frames.md b/InternalDocs/frames.md index 804d7436018..60ab2055afa 100644 --- a/InternalDocs/frames.md +++ b/InternalDocs/frames.md @@ -111,6 +111,55 @@ ### Shim frames instruction which cleans up the shim frame and returns. +### Base frame + +Each thread state contains an embedded `_PyInterpreterFrame` called the "base frame" +that serves as a sentinel at the bottom of the frame stack. This frame is allocated +in `_PyThreadStateImpl` (the internal extension of `PyThreadState`) and initialized +when the thread state is created. The `owner` field is set to `FRAME_OWNED_BY_INTERPRETER`. + +External profilers and sampling tools can validate that they have successfully unwound +the complete call stack by checking that the frame chain terminates at the base frame. +The `PyThreadState.base_frame` pointer provides the expected address to compare against. +If a stack walk doesn't reach this frame, the sample is incomplete (possibly due to a +race condition) and should be discarded. + +The base frame is embedded in `_PyThreadStateImpl` rather than `PyThreadState` because +`_PyInterpreterFrame` is defined in internal headers that cannot be exposed in the +public API. A pointer (`PyThreadState.base_frame`) is provided for profilers to access +the address without needing internal headers. + +See the initialization in `new_threadstate()` in [Python/pystate.c](../Python/pystate.c). + +#### How profilers should use the base frame + +External profilers should read `tstate->base_frame` before walking the stack, then +walk from `tstate->current_frame` following `frame->previous` pointers until reaching +a frame with `owner == FRAME_OWNED_BY_INTERPRETER`. After the walk, verify that the +last frame address matches `base_frame`. If not, discard the sample as incomplete +since the frame chain may have been in an inconsistent state due to concurrent updates. + + +### Remote Profiling Frame Cache + +The `last_profiled_frame` field in `PyThreadState` supports an optimization for +remote profilers that sample call stacks from external processes. When a remote +profiler reads the call stack, it writes the current frame address to this field. +The eval loop then keeps this pointer valid by updating it to the parent frame +whenever a frame returns (in `_PyEval_FrameClearAndPop`). + +This creates a "high-water mark" that always points to a frame still on the stack. +On subsequent samples, the profiler can walk from `current_frame` until it reaches +`last_profiled_frame`, knowing that frames from that point downward are unchanged +and can be retrieved from a cache. This significantly reduces the amount of remote +memory reads needed when call stacks are deep and stable at their base. + +The update in `_PyEval_FrameClearAndPop` is guarded: it only writes when +`last_profiled_frame` is non-NULL AND matches the frame being popped. This +prevents transient frames (called and returned between profiler samples) from +corrupting the cache pointer, while avoiding any overhead when profiling is inactive. + + ### The Instruction Pointer `_PyInterpreterFrame` has two fields which are used to maintain the instruction diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md index a7d872f3ec4..0ef45ff8e02 100644 --- a/InternalDocs/garbage_collector.md +++ b/InternalDocs/garbage_collector.md @@ -107,7 +107,7 @@ [Optimization: reusing fields to save memory](#optimization-reusing-fields-to-save-memory) section, these two extra fields are normally used to keep doubly linked lists of all the objects tracked by the garbage collector (these lists are the GC generations, more on -that in the [Optimization: incremental collection](#Optimization-incremental-collection) section), but +that in the [Optimization: generations](#Optimization-generations) section), but they are also reused to fulfill other purposes when the full doubly linked list structure is not needed as a memory optimization. @@ -153,7 +153,11 @@ The garbage collector also temporarily repurposes the `ob_tid` (thread ID) and `ob_ref_local` (local reference count) fields for other purposes during -collections. +collections. The `ob_tid` field is later restored from the containing +mimalloc segment data structure. In some cases, such as when the original +allocating thread exits, this can result in a different `ob_tid` value. +Code should not rely on `ob_tid` being stable across operations that may +trigger garbage collection. C APIs @@ -199,22 +203,22 @@ ```pycon >>> import gc ->>> +>>> >>> class Link: ... def __init__(self, next_link=None): ... self.next_link = next_link -... +... >>> link_3 = Link() >>> link_2 = Link(link_3) >>> link_1 = Link(link_2) >>> link_3.next_link = link_1 >>> A = link_1 >>> del link_1, link_2, link_3 ->>> +>>> >>> link_4 = Link() >>> link_4.next_link = link_4 >>> del link_4 ->>> +>>> >>> # Collect the unreachable Link object (and its .__dict__ dict). >>> gc.collect() 2 @@ -356,11 +360,12 @@ the reference counts fall to 0, triggering the destruction of all unreachable objects. -Optimization: incremental collection -==================================== +Optimization: generations +========================= -In order to bound the length of each garbage collection pause, the GC implementation -for the default build uses incremental collection with two generations. +In order to limit the time each garbage collection takes, the GC +implementation for the default build uses a popular optimization: +generations. Generational garbage collection takes advantage of what is known as the weak generational hypothesis: Most objects die young. @@ -368,76 +373,29 @@ programs as many temporary objects are created and destroyed very quickly. To take advantage of this fact, all container objects are segregated into -two generations: young and old. Every new object starts in the young generation. -Each garbage collection scans the entire young generation and part of the old generation. - -The time taken to scan the young generation can be controlled by controlling its -size, but the size of the old generation cannot be controlled. -In order to keep pause times down, scanning of the old generation of the heap -occurs in increments. - -To keep track of what has been scanned, the old generation contains two lists: - -* Those objects that have not yet been scanned, referred to as the `pending` list. -* Those objects that have been scanned, referred to as the `visited` list. - -To detect and collect all unreachable objects in the heap, the garbage collector -must scan the whole heap. This whole heap scan is called a full scavenge. - -Increments ----------- - -Each full scavenge is performed in a series of increments. -For each full scavenge, the combined increments will cover the whole heap. - -Each increment is made up of: - -* The young generation -* The old generation's least recently scanned objects -* All objects reachable from those objects that have not yet been scanned this full scavenge - -The surviving objects (those that are not collected) are moved to the back of the -`visited` list in the old generation. - -When a full scavenge starts, no objects in the heap are considered to have been scanned, -so all objects in the old generation must be in the `pending` space. -When all objects in the heap have been scanned a cycle ends, and all objects are moved -to the `pending` list again. To avoid having to traverse the entire list, which list is -`pending` and which is `visited` is determined by a field in the `GCState` struct. -The `visited` and `pending` lists can be swapped by toggling this bit. - -Correctness ------------ - -The [algorithm for identifying cycles](#Identifying-reference-cycles) will find all -unreachable cycles in a list of objects, but will not find any cycles that are -even partly outside of that list. -Therefore, to be guaranteed that a full scavenge will find all unreachable cycles, -each cycle must be fully contained within a single increment. - -To make sure that no partial cycles are included in the increment we perform a -[transitive closure](https://en.wikipedia.org/wiki/Transitive_closure) -over reachable, unscanned objects from the initial increment. -Since the transitive closure of objects reachable from an object must be a (non-strict) -superset of any unreachable cycle including that object, we are guaranteed that a -transitive closure cannot contain any partial cycles. -We can exclude scanned objects, as they must have been reachable when scanned. -If a scanned object becomes part of an unreachable cycle after being scanned, it will -not be collected at this time, but it will be collected in the next full scavenge. +three spaces/generations. Every new +object starts in the first generation (generation 0). The previous algorithm is +executed only over the objects of a particular generation and if an object +survives a collection of its generation it will be moved to the next one +(generation 1), where it will be surveyed for collection less often. If +the same object survives another GC round in this new generation (generation 1) +it will be moved to the last generation (generation 2) where it will be +surveyed the least often. > [!NOTE] -> The GC implementation for the free-threaded build does not use incremental collection. -> Every collection operates on the entire heap. +> The GC implementation for the free-threaded build does not use generational +> collection. Every collection operates on the entire heap. + In order to decide when to run, the collector keeps track of the number of object allocations and deallocations since the last collection. When the number of allocations minus the number of deallocations exceeds `threshold0`, -collection starts. `threshold1` determines the fraction of the old -collection that is included in the increment. -The fraction is inversely proportional to `threshold1`, -as historically a larger `threshold1` meant that old generation -collections were performed less frequently. -`threshold2` is ignored. +collection starts. Initially only generation 0 is examined. If generation 0 has +been examined more than `threshold_1` times since generation 1 has been +examined, then generation 1 is examined as well. With generation 2, +things are a bit more complicated; see +[Collecting the oldest generation](#Collecting-the-oldest-generation) for +more information. These thresholds can be examined using the [`gc.get_threshold()`](https://docs.python.org/3/library/gc.html#gc.get_threshold) @@ -446,7 +404,7 @@ ```pycon >>> import gc >>> gc.get_threshold() -(700, 10, 10) +(2000, 10, 10) ``` The content of these generations can be examined using the @@ -459,84 +417,61 @@ ... pass ... >>> # Move everything to the old generation so it's easier to inspect ->>> # the young generation. +>>> # the younger generation. >>> gc.collect() 0 >>> # Create a reference cycle. >>> x = MyObj() >>> x.self = x ->>> ->>> # Initially the object is in the young generation. +>>> +>>> # Initially the object is in the youngest generation. >>> gc.get_objects(generation=0) [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] ->>> +>>> >>> # After a collection of the youngest generation the object ->>> # moves to the old generation. +>>> # moves to the next generation. >>> gc.collect(generation=0) 0 >>> gc.get_objects(generation=0) [] >>> gc.get_objects(generation=1) -[] ->>> gc.get_objects(generation=2) [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] ``` +Collecting the oldest generation +-------------------------------- + +In addition to the various configurable thresholds, the GC only triggers a full +collection of the oldest generation if the ratio `long_lived_pending / long_lived_total` +is above a given value (hardwired to 25%). The reason is that, while "non-full" +collections (that is, collections of the young and middle generations) will always +examine roughly the same number of objects (determined by the aforementioned +thresholds) the cost of a full collection is proportional to the total +number of long-lived objects, which is virtually unbounded. Indeed, it has +been remarked that doing a full collection every of object +creations entails a dramatic performance degradation in workloads which consist +of creating and storing lots of long-lived objects (for example, building a large list +of GC-tracked objects would show quadratic performance, instead of linear as +expected). Using the above ratio, instead, yields amortized linear performance +in the total number of objects (the effect of which can be summarized thusly: +"each full garbage collection is more and more costly as the number of objects +grows, but we do fewer and fewer of them"). + Optimization: excluding reachable objects ========================================= An object cannot be garbage if it can be reached. To avoid having to identify -reference cycles across the whole heap, we can reduce the amount of work done -considerably by first identifying objects reachable from objects known to be -alive. These objects are excluded from the normal cyclic detection process. - -The default and free-threaded build both implement this optimization but in -slightly different ways. - -Finding reachable objects for the default build GC --------------------------------------------------- - -This works by first moving most reachable objects to the `visited` space. -Empirically, most reachable objects can be reached from a small set of global -objects and local variables. This step does much less work per object, so -reduces the time spent performing garbage collection by at least half. - -> [!NOTE] -> Objects that are not determined to be reachable by this pass are not necessarily -> unreachable. We still need to perform the main algorithm to determine which objects -> are actually unreachable. -We use the same technique of forming a transitive closure as the incremental -collector does to find reachable objects, seeding the list with some global -objects and the currently executing frames. - -This phase moves objects to the `visited` space, as follows: - -1. All objects directly referred to by any builtin class, the `sys` module, the `builtins` -module and all objects directly referred to from stack frames are added to a working -set of reachable objects. -2. Until this working set is empty: - 1. Pop an object from the set and move it to the `visited` space - 2. For each object directly reachable from that object: - * If it is not already in `visited` space and it is a GC object, - add it to the working set - - -Before each increment of collection is performed, the stacks are scanned -to check for any new stack frames that have been created since the last -increment. All objects directly referred to from those stack frames are -added to the working set. -Then the above algorithm is repeated, starting from step 2. - +reference cycles across the whole heap, the free-threaded build first identifies +objects reachable from objects known to be alive. These objects are excluded +from the normal cyclic detection process. Finding reachable objects for the free-threaded GC -------------------------------------------------- Within the `gc_free_threading.c` implementation, this is known as the "mark -alive" pass or phase. It is similar in concept to what is done for the default -build GC. Rather than moving objects between double-linked lists, the -free-threaded GC uses a flag in `ob_gc_bits` to track if an object is -found to be definitely alive (not garbage). +alive" pass or phase. The free-threaded GC uses a flag in `ob_gc_bits` to track +if an object is found to be definitely alive (not garbage). To find objects reachable from known alive objects, known as the "roots", the `gc_mark_alive_from_roots()` function is used. Root objects include @@ -767,6 +702,14 @@ already not tracked. Tuples are examined for untracking in all garbage collection cycles. +Dictionaries are always tracked from creation and are not untracked by the +garbage collector. Earlier versions (up to 3.13) used lazy tracking: empty or +atomic-only dicts were untracked on creation and re-tracked when a trackable +value was inserted (via `MAINTAIN_TRACKING`), and full collections called +`_PyDict_MaybeUntrack` to prune dicts whose values had become atomic. That +machinery was removed in 3.14 (GH-127010) because the per-set-item cost of +checking the tracking invariant outweighed the savings on full collections. + The garbage collector module provides the Python function `is_tracked(obj)`, which returns the current tracking status of the object. Subsequent garbage collections may change the tracking status of the object. diff --git a/InternalDocs/generators.md b/InternalDocs/generators.md index 979a5b51521..fa652cd231c 100644 --- a/InternalDocs/generators.md +++ b/InternalDocs/generators.md @@ -64,7 +64,7 @@ The [`FOR_ITER`](https://docs.python.org/dev/library/dis.html#opcode-FOR_ITER) instruction calls `__next__` on the iterator which is on the top of the stack, -and pushes the result to the stack. It has [`specializations`](adaptive.md) +and pushes the result to the stack. It has [`specializations`](interpreter.md) for a few common iterator types, including `FOR_ITER_GEN`, for iterating over a generator. `FOR_ITER_GEN` bypasses the call to `__next__`, and instead directly pushes the generator stack and resumes its execution from the diff --git a/InternalDocs/interpreter.md b/InternalDocs/interpreter.md index 38e9f6fced6..7fc41a807dd 100644 --- a/InternalDocs/interpreter.md +++ b/InternalDocs/interpreter.md @@ -226,10 +226,11 @@ ## The call stack heap allocation for the stack frame. Since 3.11, frames are no longer fully-fledged objects. Instead, a leaner internal -`_PyInterpreterFrame` structure is used, which is allocated using a custom allocator -function (`_PyThreadState_BumpFramePointer()`), which allocates and initializes a -frame structure. Usually a frame allocation is just a pointer bump, which improves -memory locality. +`_PyInterpreterFrame` structure is used. Most frames are allocated contiguously in a +per-thread stack (see `_PyThreadState_PushFrame` in [Python/pystate.c](../Python/pystate.c)), +which improves memory locality and reduces overhead. +If the current `datastack_chunk` has enough space (`_PyThreadState_HasStackSpace`) +then the lightweight `_PyFrame_PushUnchecked` can be used instead of `_PyThreadState_PushFrame`. Sometimes an actual `PyFrameObject` is needed, such as when Python code calls `sys._getframe()` or an extension module calls @@ -506,6 +507,38 @@ ### Maintaining stats After an optimization has been deferred in the adaptive instruction, that should be recorded with `STAT_INC(BASE_INSTRUCTION, deferred)`. +## Interpreter types +There are three different types of interpreters to choose from based on compiler support: + + * traditional switch-case interpreter + + Supported by all compilers covered in PEP 7. + + * computed-gotos interpreter + + Enabled using configure option `--with-computed-gotos` and used by default on supported compilers. + It uses [Labels as Values](https://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html) + for more efficient dispatching. + + * tail-calling interpreter + + Enabled using configure option `--with-tail-call-interp` (or `--tail-call-interp` for build.bat on Windows). + It uses [tail calls](https://clang.llvm.org/docs/AttributeReference.html#musttail) and the + [preserve_none](https://clang.llvm.org/docs/AttributeReference.html#preserve-none) + calling convention between the small C functions that implement individual Python opcodes. + + Not all compilers support these and if they do not all targets might be supported (for example, + MSVC currently only supports x64 and only in optimized builds). + + In addition, compilers must do [escape analysis](https://gcc.gnu.org/onlinedocs/gcc/Common-Attributes.html#index-musttail) + of the lifetimes of automatic variables, function parameters, and temporaries to ensure proper tail-calls. They + emit a compile error in case of a violation or detection failure. The ability to detect this varies depending on the compiler and + also on the optimization level. Following techniques are particularly helpful to the MSVC compiler in this regard + * [Introducing additional scopes](https://github.com/python/cpython/blob/3908593039bde9d4b591ab09919003ee57418d64/Python/bytecodes.c#L2526) + * [extracting problematic code paths into a separate function](https://github.com/python/cpython/pull/143068/files#diff-729a985b0cb8b431cb291f1edb561bbbfea22e3f8c262451cd83328a0936a342R3724) + * [returning a pointer instead of taking it as an output parameter](https://github.com/python/cpython/blob/3908593039bde9d4b591ab09919003ee57418d64/Include/internal/pycore_ceval.h#L489-L492) + + Using `restrict` is another (currently unused) remedy. Additional resources -------------------- diff --git a/InternalDocs/jit.md b/InternalDocs/jit.md index c98ccb3ace8..1345f2db8b3 100644 --- a/InternalDocs/jit.md +++ b/InternalDocs/jit.md @@ -11,24 +11,54 @@ # The JIT Historically, the adaptive interpreter was referred to as `tier 1` and the JIT as `tier 2`. You will see remnants of this in the code. -## The Optimizer and Executors +## The Trace Recorder and Executors -The program begins running on the adaptive interpreter, until a `JUMP_BACKWARD` -instruction determines that it is "hot" because the counter in its +There are two interpreters in this section: + 1. Adaptive interpreter (the default behavior) + 2. Trace recording interpreter (enabled on JIT builds) + +The program begins running on the adaptive interpreter, until a `JUMP_BACKWARD` or +`RESUME` instruction determines that it is "hot" because the counter in its [inline cache](interpreter.md#inline-cache-entries) indicates that it executed more than some threshold number of times (see [`backoff_counter_triggers`](../Include/internal/pycore_backoff.h)). -It then calls the function `_PyOptimizer_Optimize()` in +It then calls the function `_PyJit_TryInitializeTracing` in [`Python/optimizer.c`](../Python/optimizer.c), passing it the current -[frame](frames.md) and instruction pointer. `_PyOptimizer_Optimize()` -constructs an object of type -[`_PyExecutorObject`](../Include/internal/pycore_optimizer.h) which implements -an optimized version of the instruction trace beginning at this jump. +[frame](frames.md), instruction pointer and state. +The interpreter then switches into "tracing mode" via the macro +`ENTER_TRACING()`. On platforms that support computed goto and tail-calling +interpreters, the dispatch table is swapped out, while other platforms that do +not support either use a single flag in the opcode. +Execution between the normal interpreter and tracing interpreter are +interleaved via this dispatch mechanism. This means that while logically +there are two interpreters, the implementation appears to be a single +interpreter. -The optimizer determines where the trace ends, and the executor is set up +During tracing mode, after each interpreter instruction's `DISPATCH()`, +the interpreter jumps to the `TRACE_RECORD` instruction. This instruction +records the previous instruction executed and also any live values of the next +operation it may require. It then translates the previous instruction to +a sequence of micro-ops using `_PyJit_translate_single_bytecode_to_trace`. +To ensure that the adaptive interpreter instructions +and cache entries are up-to-date, the trace recording interpreter always resets +the adaptive counters of adaptive instructions it sees. +This forces a re-specialization of any new instruction should an instruction +deoptimize. Thus, feeding the trace recorder up-to-date information. +Finally, the `TRACE_RECORD` instruction decides when to stop tracing +using various heuristics. + +Once trace recording concludes, `LEAVE_TRACING()` swaps out the dispatch +table/the opcode flag set earlier by `ENTER_TRACING()` is unset. +`stop_tracing_and_jit()` then calls `_PyOptimizer_Optimize()` which optimizes +the trace and constructs an +[`_PyExecutorObject`](../Include/internal/pycore_optimizer.h). + +JIT execution is set up to either return to the adaptive interpreter and resume execution, or transfer control to another executor (see `_PyExitData` in -Include/internal/pycore_optimizer.h). +Include/internal/pycore_optimizer.h). When resuming to the adaptive interpreter, +a "side exit", generated by an `EXIT_IF` may trigger recording of another trace. +While a "deopt", generated by a `DEOPT_IF`, does not trigger recording. The executor is stored on the [`code object`](code_objects.md) of the frame, in the `co_executors` field which is an array of executors. The start @@ -40,12 +70,7 @@ ## The micro-op optimizer The micro-op (abbreviated `uop` to approximate `μop`) optimizer is defined in [`Python/optimizer.c`](../Python/optimizer.c) as `_PyOptimizer_Optimize`. -It translates an instruction trace into a sequence of micro-ops by replacing -each bytecode by an equivalent sequence of micro-ops (see -`_PyOpcode_macro_expansion` in -[pycore_opcode_metadata.h](../Include/internal/pycore_opcode_metadata.h) -which is generated from [`Python/bytecodes.c`](../Python/bytecodes.c)). -The micro-op sequence is then optimized by +It takes a micro-op sequence from the trace recorder and optimizes with `_Py_uop_analyze_and_optimize` in [`Python/optimizer_analysis.c`](../Python/optimizer_analysis.c) and an instance of `_PyUOpExecutor_Type` is created to contain it. @@ -53,7 +78,7 @@ ## The micro-op optimizer ## The JIT interpreter After a `JUMP_BACKWARD` instruction invokes the uop optimizer to create a uop -executor, it transfers control to this executor via the `GOTO_TIER_TWO` macro. +executor, it transfers control to this executor via the `TIER1_TO_TIER2` macro. CPython implements two executors. Here we describe the JIT interpreter, which is the simpler of them and is therefore useful for debugging and analyzing @@ -78,8 +103,8 @@ ## The JIT interpreter ## Invalidating Executors In addition to being stored on the code object, each executor is also -inserted into a list of all executors, which is stored in the interpreter -state's `executor_list_head` field. This list is used when it is necessary +inserted into contiguous arrays (`executor_blooms` and `executor_ptrs`) +stored in the interpreter state. These arrays are used when it is necessary to invalidate executors because values they used in their construction may have changed. @@ -89,7 +114,7 @@ ## The JIT [`--enable-experimental-jit`](https://docs.python.org/dev/using/configure.html#cmdoption-enable-experimental-jit), the uop executor's `jit_code` field is populated with a pointer to a compiled C function that implements the executor logic. This function's signature is -defined by `jit_func` in [`pycore_jit.h`](Include/internal/pycore_jit.h). +defined by `jit_func` in [`pycore_jit.h`](../Include/internal/pycore_jit.h). When the executor is invoked by `ENTER_EXECUTOR`, instead of jumping to the uop interpreter at `tier2_dispatch`, the executor runs the function that `jit_code` points to. This function returns the instruction pointer diff --git a/InternalDocs/parser.md b/InternalDocs/parser.md index 1d0ffe6d40d..1bb4cdea543 100644 --- a/InternalDocs/parser.md +++ b/InternalDocs/parser.md @@ -819,6 +819,13 @@ $ python -m pegen python ``` +> [!CAUTION] +> Python's grammar (the `Grammar/python.gram` file) is written for the +> C backend. To experiment, you will need to write a grammar +> without C-specific parts like actions and the trailer. +> See [#133560](https://github.com/python/cpython/issues/133560) +> and [#96424](https://github.com/python/cpython/issues/96424) for more information. + This will generate a file called `parse.py` in the same directory that you can use to parse some input: diff --git a/InternalDocs/profiling_binary_format.md b/InternalDocs/profiling_binary_format.md new file mode 100644 index 00000000000..7e4592a0d89 --- /dev/null +++ b/InternalDocs/profiling_binary_format.md @@ -0,0 +1,541 @@ +# Profiling Binary Format + +The profiling module includes a binary file format for storing sampling +profiler data. This document describes the format's structure and the +design decisions behind it. + +The implementation is in +[`Modules/_remote_debugging/binary_io_writer.c`](../Modules/_remote_debugging/binary_io_writer.c) +and [`Modules/_remote_debugging/binary_io_reader.c`](../Modules/_remote_debugging/binary_io_reader.c), +with declarations in +[`Modules/_remote_debugging/binary_io.h`](../Modules/_remote_debugging/binary_io.h). + +## Overview + +The sampling profiler can generate enormous amounts of data. A typical +profiling session sampling at 1000 Hz for 60 seconds produces 60,000 samples. +Each sample contains a full call stack, often 20-50 frames deep, and each +frame includes a filename, function name, and line number. In a text-based +format like collapsed stacks, this would mean repeating the same long file +paths and function names thousands of times. + +The binary format addresses this through two key strategies: + +1. **Deduplication**: Strings and frames are stored once in lookup tables, + then referenced by small integer indices. A 100-character file path that + appears in 50,000 samples is stored once, not 50,000 times. + +2. **Compact encoding**: Variable-length integers (varints) encode small + values in fewer bytes. Since most indices are small (under 128), they + typically need only one byte instead of four. + +Together with optional zstd compression, these techniques reduce file sizes +by 10-50x compared to text formats while also enabling faster I/O. + +## File Layout + +The file consists of five sections: + +``` ++------------------+ Offset 0 +| Header | 64 bytes (fixed) ++------------------+ Offset 64 +| | +| Sample Data | Variable size (optionally compressed) +| | ++------------------+ string_table_offset +| String Table | Variable size ++------------------+ frame_table_offset +| Frame Table | Variable size ++------------------+ file_size - 32 +| Footer | 32 bytes (fixed) ++------------------+ file_size +``` + +The layout is designed for streaming writes during profiling. The profiler +cannot know in advance how many unique strings or frames will be encountered, +so these tables must be built incrementally and written at the end. + +The header comes first so readers can quickly validate the file and locate +the metadata tables. The sample data follows immediately, allowing the writer +to stream samples directly to disk (or through a compression stream) without +buffering the entire dataset in memory. + +The string and frame tables are placed after sample data because they grow +as new unique entries are discovered during profiling. By deferring their +output until finalization, the writer avoids the complexity of reserving +space or rewriting portions of the file. + +The footer at the end contains counts needed to allocate arrays before +parsing the tables. Placing it at a fixed offset from the end (rather than +at a variable offset recorded in the header) means readers can locate it +with a single seek to `file_size - 32`, without first reading the header. + +## Header + +``` + Offset Size Type Description ++--------+------+---------+----------------------------------------+ +| 0 | 4 | uint32 | Magic number (0x54414348 = "TACH") | +| 4 | 4 | uint32 | Format version | +| 8 | 4 | bytes | Python version (major, minor, micro, | +| | | | reserved) | +| 12 | 8 | uint64 | Start timestamp (microseconds) | +| 20 | 8 | uint64 | Sample interval (microseconds) | +| 28 | 4 | uint32 | Total sample count | +| 32 | 4 | uint32 | Thread count | +| 36 | 8 | uint64 | String table offset | +| 44 | 8 | uint64 | Frame table offset | +| 52 | 4 | uint32 | Compression type (0=none, 1=zstd) | +| 56 | 8 | bytes | Reserved (zero-filled) | ++--------+------+---------+----------------------------------------+ +``` + +The magic number `0x54414348` ("TACH" for Tachyon) identifies the file format +and also serves as an **endianness marker**. When read on a system with +different byte order than the writer, it appears as `0x48434154`. The reader +uses this to detect cross-endian files and automatically byte-swap all +multi-byte integer fields. + +The Python version field records the major, minor, and micro version numbers +of the Python interpreter that generated the file. This allows analysis tools +to detect version mismatches when replaying data collected on a different +Python version, which may have different internal structures or behaviors. + +The header is written as zeros initially, then overwritten with actual values +during finalization. This requires the output stream to be seekable, which +is acceptable since the format targets regular files rather than pipes or +network streams. + +## Sample Data + +Sample data begins at offset 64 and extends to `string_table_offset`. Samples +use delta compression to minimize redundancy when consecutive samples from the +same thread have identical or similar call stacks. + +### Stack Encoding Types + +Each sample record begins with thread identification, then an encoding byte: + +| Code | Name | Description | +|------|------|-------------| +| 0x00 | REPEAT | RLE: identical stack repeated N times | +| 0x01 | FULL | Complete stack (first sample or no match) | +| 0x02 | SUFFIX | Shares N frames from bottom of previous stack | +| 0x03 | POP_PUSH | Remove M frames from top, add N new frames | + +### Record Formats + +**REPEAT (0x00) - Run-Length Encoded Identical Stacks:** +``` ++-----------------+-----------+----------------------------------------+ +| thread_id | 8 bytes | Thread identifier (uint64, fixed) | +| interpreter_id | 4 bytes | Interpreter ID (uint32, fixed) | +| encoding | 1 byte | 0x00 (REPEAT) | +| count | varint | Number of samples in this RLE group | +| samples | varies | Interleaved: [delta: varint, status: 1]| +| | | repeated count times | ++-----------------+-----------+----------------------------------------+ +``` +The stack is inherited from this thread's previous sample. Each sample in the +group gets its own timestamp delta and status byte, stored as interleaved pairs +(delta1, status1, delta2, status2, ...) rather than separate arrays. + +**FULL (0x01) - Complete Stack:** +``` ++-----------------+-----------+----------------------------------------+ +| thread_id | 8 bytes | Thread identifier (uint64, fixed) | +| interpreter_id | 4 bytes | Interpreter ID (uint32, fixed) | +| encoding | 1 byte | 0x01 (FULL) | +| timestamp_delta | varint | Microseconds since thread's last sample| +| status | 1 byte | Thread state flags | +| stack_depth | varint | Number of frames in call stack | +| frame_indices | varint[] | Array of frame table indices | ++-----------------+-----------+----------------------------------------+ +``` +Used for the first sample from a thread, or when delta encoding would not +provide savings. + +**SUFFIX (0x02) - Shared Suffix Match:** +``` ++-----------------+-----------+----------------------------------------+ +| thread_id | 8 bytes | Thread identifier (uint64, fixed) | +| interpreter_id | 4 bytes | Interpreter ID (uint32, fixed) | +| encoding | 1 byte | 0x02 (SUFFIX) | +| timestamp_delta | varint | Microseconds since thread's last sample| +| status | 1 byte | Thread state flags | +| shared_count | varint | Frames shared from bottom of prev stack| +| new_count | varint | New frames at top of stack | +| new_frames | varint[] | Array of new_count frame indices | ++-----------------+-----------+----------------------------------------+ +``` +Used when a function call added frames to the top of the stack. The shared +frames from the previous stack are kept, and new frames are prepended. + +**POP_PUSH (0x03) - Pop and Push:** +``` ++-----------------+-----------+----------------------------------------+ +| thread_id | 8 bytes | Thread identifier (uint64, fixed) | +| interpreter_id | 4 bytes | Interpreter ID (uint32, fixed) | +| encoding | 1 byte | 0x03 (POP_PUSH) | +| timestamp_delta | varint | Microseconds since thread's last sample| +| status | 1 byte | Thread state flags | +| pop_count | varint | Frames to remove from top of prev stack| +| push_count | varint | New frames to add at top | +| new_frames | varint[] | Array of push_count frame indices | ++-----------------+-----------+----------------------------------------+ +``` +Used when the code path changed: some frames were popped (function returns) +and new frames were pushed (different function calls). + +### Thread and Interpreter Identification + +Thread IDs are 64-bit values that can be large (memory addresses on some +platforms) and vary unpredictably. Using a fixed 8-byte encoding avoids +the overhead of varint encoding for large values and simplifies parsing +since the reader knows exactly where each field begins. + +The interpreter ID identifies which Python sub-interpreter the thread +belongs to, allowing analysis tools to separate activity across interpreters +in processes using multiple sub-interpreters. + +### Status Byte + +The status byte is a bitfield encoding thread state at sample time: + +| Bit | Flag | Meaning | +|-----|-----------------------|--------------------------------------------| +| 0 | THREAD_STATUS_HAS_GIL | Thread holds the GIL (Global Interpreter Lock) | +| 1 | THREAD_STATUS_ON_CPU | Thread is actively running on a CPU core | +| 2 | THREAD_STATUS_UNKNOWN | Thread state could not be determined | +| 3 | THREAD_STATUS_GIL_REQUESTED | Thread is waiting to acquire the GIL | +| 4 | THREAD_STATUS_HAS_EXCEPTION | Thread has a pending exception | + +Multiple flags can be set simultaneously (e.g., a thread can hold the GIL +while also running on CPU). Analysis tools use these to filter samples or +visualize thread states over time. + +### Timestamp Delta Encoding + +Timestamps use delta encoding rather than absolute values. Absolute +timestamps in microseconds require 8 bytes each, but consecutive samples +from the same thread are typically separated by the sampling interval +(e.g., 1000 microseconds), so the delta between them is small and fits +in 1-2 varint bytes. The writer tracks the previous timestamp for each +thread separately. The first sample from a thread encodes its delta from +the profiling start time; subsequent samples encode the delta from that +thread's previous sample. This per-thread tracking is necessary because +samples are interleaved across threads in arrival order, not grouped by +thread. + +For REPEAT (RLE) records, timestamp deltas and status bytes are stored as +interleaved pairs (delta, status, delta, status, ...) - one pair per +repeated sample - allowing efficient batching while preserving the exact +timing and state of each sample. + +### Frame Indexing + +Each frame in a call stack is represented by an index into the frame table +rather than inline data. This provides massive space savings because call +stacks are highly repetitive: the same function appears in many samples +(hot functions), call stacks often share common prefixes (main -> app -> +handler -> ...), and recursive functions create repeated frame sequences. +A frame index is typically 1-2 varint bytes. Inline frame data would be +20-200+ bytes (two strings plus a line number). For a profile with 100,000 +samples averaging 30 frames each, this reduces frame data from potentially +gigabytes to tens of megabytes. + +Frame indices are written innermost-first (the currently executing frame +has index 0 in the array). This ordering works well with delta compression: +function calls typically add frames at the top (index 0), while shared +frames remain at the bottom. + +## String Table + +The string table stores deduplicated UTF-8 strings (filenames and function +names). It begins at `string_table_offset` and contains entries in order of +their assignment during writing: + +``` ++----------------+ +| length: varint | +| data: bytes | ++----------------+ (repeated for each string) +``` + +Strings are stored in the order they were first encountered during writing. +The first unique filename gets index 0, the second gets index 1, and so on. +Length-prefixing (rather than null-termination) allows strings containing +null bytes and enables readers to allocate exact-sized buffers. The varint +length encoding means short strings (under 128 bytes) need only one length +byte. + +## Frame Table + +The frame table stores deduplicated frame entries with full source position +information and bytecode opcode: + +``` ++----------------------------+ +| filename_idx: varint | +| funcname_idx: varint | +| lineno: svarint | +| end_lineno_delta: svarint | +| column: svarint | +| end_column_delta: svarint | +| opcode: u8 | ++----------------------------+ (repeated for each frame) +``` + +### Field Definitions + +| Field | Type | Description | +|------------------|---------------|----------------------------------------------------------| +| filename_idx | varint | Index into string table for file name | +| funcname_idx | varint | Index into string table for function name | +| lineno | zigzag varint | Start line number (-1 for synthetic frames) | +| end_lineno_delta | zigzag varint | Delta from lineno (end_lineno = lineno + delta) | +| column | zigzag varint | Start column offset in UTF-8 bytes (-1 if not available) | +| end_column_delta | zigzag varint | Delta from column (end_column = column + delta) | +| opcode | u8 | Python bytecode opcode (0-254) or 255 for None | + +### Delta Encoding + +Position end values use delta encoding for efficiency: + +- `end_lineno = lineno + end_lineno_delta` +- `end_column = column + end_column_delta` + +Typical values: +- `end_lineno_delta`: Usually 0 (single-line expressions) → encodes to 1 byte +- `end_column_delta`: Usually 5-20 (expression width) → encodes to 1 byte + +This saves ~1-2 bytes per frame compared to absolute encoding. When the base +value (lineno or column) is -1 (not available), the delta is stored as 0 and +the reconstructed value is -1. + +### Sentinel Values + +- `opcode = 255`: No opcode captured +- `lineno = -1`: Synthetic frame (no source location) +- `column = -1`: Column offset not available + +### Deduplication + +Each unique (filename, funcname, lineno, end_lineno, column, end_column, +opcode) combination gets one entry. This enables instruction-level profiling +where multiple bytecode instructions on the same line can be distinguished. + +Strings and frames are deduplicated separately because they have different +cardinalities and reference patterns. A codebase might have hundreds of +unique source files but thousands of unique functions. Many functions share +the same filename, so storing the filename index in each frame entry (rather +than the full string) provides an additional layer of deduplication. A frame +entry is typically 7-9 bytes rather than two full strings plus location data. + +### Size Analysis + +Typical frame size with delta encoding: +- file_idx: 1-2 bytes +- func_idx: 1-2 bytes +- lineno: 1-2 bytes +- end_lineno_delta: 1 byte (usually 0) +- column: 1 byte (usually < 64) +- end_column_delta: 1 byte (usually < 64) +- opcode: 1 byte + +**Total: ~7-9 bytes per frame** + +Line numbers and columns use signed varint (zigzag encoding) to handle +sentinel values efficiently. Synthetic frames—generated frames that don't +correspond directly to Python source code, such as C extension boundaries or +internal interpreter frames—use -1 to indicate the absence of a source +location. Zigzag encoding ensures these small negative values encode +efficiently (−1 becomes 1, which is one byte) rather than requiring the +maximum varint length. + +## Footer + +``` + Offset Size Type Description ++--------+------+---------+----------------------------------------+ +| 0 | 4 | uint32 | String count | +| 4 | 4 | uint32 | Frame count | +| 8 | 8 | uint64 | Total file size | +| 16 | 16 | bytes | Checksum (reserved, currently zeros) | ++--------+------+---------+----------------------------------------+ +``` + +The string and frame counts allow readers to pre-allocate arrays of the +correct size before parsing the tables. Without these counts, readers would +need to either scan the tables twice (once to count, once to parse) or use +dynamically-growing arrays. + +The file size field provides a consistency check: if the actual file size +does not match, the file may be truncated or corrupted. + +The checksum field is reserved for future use. A checksum would allow +detection of corruption but adds complexity and computation cost. The +current implementation leaves this as zeros. + +## Variable-Length Integer Encoding + +The format uses LEB128 (Little Endian Base 128) for unsigned integers and +zigzag + LEB128 for signed integers. These encodings are widely used +(Protocol Buffers, DWARF debug info, WebAssembly) and well-understood. + +### Unsigned Varint (LEB128) + +Each byte stores 7 bits of data. The high bit indicates whether more bytes +follow: + +``` +Value Encoded bytes +0-127 [0xxxxxxx] (1 byte) +128-16383 [1xxxxxxx] [0xxxxxxx] (2 bytes) +16384+ [1xxxxxxx] [1xxxxxxx] ... (3+ bytes) +``` + +Most indices in profiling data are small. A profile with 1000 unique frames +needs at most 2 bytes per frame index. The common case (indices under 128) +needs only 1 byte. + +### Signed Varint (Zigzag) + +Standard LEB128 encodes −1 as a very large unsigned value, requiring many +bytes. Zigzag encoding interleaves positive and negative values: + +``` + 0 -> 0 -1 -> 1 1 -> 2 -2 -> 3 2 -> 4 +``` + +This ensures small-magnitude values (whether positive or negative) encode +in few bytes. + +## Compression + +When compression is enabled, the sample data region contains a zstd stream. +The string table, frame table, and footer remain uncompressed so readers can +access metadata without decompressing the entire file. A tool that only needs +to report "this file contains 50,000 samples of 3 threads" can read the header +and footer without touching the compressed sample data. This also simplifies +the format: the header's offset fields point directly to the tables rather +than to positions within a decompressed stream. + +Zstd provides an excellent balance of compression ratio and speed. Profiling +data compresses very well (often 5-10x) due to repetitive patterns: the same +small set of frame indices appears repeatedly, and delta-encoded timestamps +cluster around the sampling interval. Zstd's streaming API allows compression +without buffering the entire dataset. The writer feeds sample data through +the compressor incrementally, flushing compressed chunks to disk as they +become available. + +Level 5 compression is used as a default. Lower levels (1-3) are faster but +compress less; higher levels (6+) compress more but slow down writing. Level +5 provides good compression with minimal impact on profiling overhead. + +## Reading and Writing + +### Writing + +1. Open the output file and write 64 zero bytes as a placeholder header +2. Initialize empty string and frame dictionaries for deduplication +3. For each sample: + - Intern any new strings, assigning sequential indices + - Intern any new frames, assigning sequential indices + - Encode the sample record and write to the buffer + - Flush the buffer through compression (if enabled) when full +4. Flush remaining buffered data and finalize compression +5. Write the string table (length-prefixed strings in index order) +6. Write the frame table (varint-encoded entries in index order) +7. Write the footer with final counts +8. Seek to offset 0 and write the header with actual values + +The writer maintains two dictionaries: one mapping strings to indices, one +mapping (filename_idx, funcname_idx, lineno) tuples to frame indices. These +enable O(1) lookup during interning. + +### Reading + +1. Read the header magic number to detect endianness (set `needs_swap` flag + if the magic appears byte-swapped) +2. Validate version and read remaining header fields (byte-swapping if needed) +3. Seek to end − 32 and read the footer (byte-swapping counts if needed) +4. Allocate string array of `string_count` elements +5. Parse the string table, populating the array +6. Allocate frame array of `frame_count * 3` uint32 elements +7. Parse the frame table, populating the array +8. If compressed, decompress the sample data region +9. Iterate through samples, resolving indices to strings/frames + (byte-swapping thread_id and interpreter_id if needed) + +The reader builds lookup arrays rather than dictionaries since it only needs +index-to-value mapping, not value-to-index. + +## Platform Considerations + +### Byte Ordering and Cross-Platform Portability + +The binary format uses **native byte order** for all multi-byte integer +fields when writing. However, the reader supports **cross-endian reading**: +files written on a little-endian system (x86, ARM) can be read on a +big-endian system (s390x, PowerPC), and vice versa. + +The magic number doubles as an endianness marker. When read on a system with +different byte order, it appears byte-swapped (`0x48434154` instead of +`0x54414348`). The reader detects this and automatically byte-swaps all +fixed-width integer fields during parsing. + +Writers must use `memcpy()` from properly-sized integer types when writing +fixed-width integer fields. When the source variable's type differs from the +field width (e.g., `size_t` written as 4 bytes), explicit casting to the +correct type (e.g., `uint32_t`) is required before `memcpy()`. On big-endian +systems, copying from an oversized type would copy the wrong bytes—high-order +zeros instead of the actual value. + +The reader tracks whether byte-swapping is needed via a `needs_swap` flag set +during header parsing. All fixed-width fields in the header, footer, and +sample data are conditionally byte-swapped using Python's internal byte-swap +functions (`_Py_bswap32`, `_Py_bswap64` from `pycore_bitutils.h`). + +Variable-length integers (varints) are byte-order independent since they +encode values one byte at a time using the LEB128 scheme, so they require +no special handling for cross-endian reading. + +### Memory-Mapped I/O + +On Unix systems (Linux, macOS), the reader uses `mmap()` to map the file +into the process address space. The kernel handles paging data in and out +as needed, no explicit read() calls or buffer management are required, +multiple readers can share the same physical pages, and sequential access +patterns benefit from kernel read-ahead. + +The implementation uses `madvise()` to hint the access pattern to the kernel: +`MADV_SEQUENTIAL` indicates the file will be read linearly, enabling +aggressive read-ahead. `MADV_WILLNEED` requests pre-faulting of pages. +On Linux, `MAP_POPULATE` pre-faults all pages at mmap time rather than on +first access, moving page fault overhead from the parsing loop to the +initial mapping for more predictable performance. For large files (over +32 MB), `MADV_HUGEPAGE` requests transparent huge pages (2 MB instead of +4 KB) to reduce TLB pressure when accessing large amounts of data. + +On Windows, the implementation falls back to standard file I/O with full +file buffering. Profiling data files are typically small enough (tens to +hundreds of megabytes) that this is acceptable. + +The writer uses a 512 KB buffer to batch small writes. Each sample record +is typically tens of bytes; writing these individually would incur excessive +syscall overhead. The buffer accumulates data until full, then flushes in +one write() call (or feeds through the compression stream). + +## Future Considerations + +The format reserves space for future extensions. The 12 reserved bytes in +the header could hold additional metadata. The 16-byte checksum field in +the footer is currently unused. The version field allows incompatible +changes with graceful rejection. New compression types could be added +(compression_type > 1). + +Any changes that alter the meaning of existing fields or the parsing logic +should increment the version number to prevent older readers from +misinterpreting new files. diff --git a/InternalDocs/stack_protection.md b/InternalDocs/stack_protection.md index 5e35b2d8195..14802e57d09 100644 --- a/InternalDocs/stack_protection.md +++ b/InternalDocs/stack_protection.md @@ -4,7 +4,7 @@ # Stack Protection Protection against pure Python stack recursion has existed since very early, but in 3.12 we added protection against stack overflow in C code. This was initially implemented using a counter and later improved in 3.14 to use the actual stack depth. For those platforms that support it (Windows, Mac, and most Linuxes) we query the operating system to find the stack bounds. -For other platforms we use conserative estimates. +For other platforms we use conservative estimates. The C stack looks like this: @@ -38,12 +38,19 @@ # Stack Protection ```python kb_used = (stack_top - stack_pointer)>>10 -if stack_pointer < hard_limit: +if stack_pointer < bottom_of_machine_stack: + pass # Our stack limits could be wrong so it is safest to do nothing. +elif stack_pointer < hard_limit: FatalError(f"Unrecoverable stack overflow (used {kb_used} kB)") elif stack_pointer < soft_limit: raise RecursionError(f"Stack overflow (used {kb_used} kB)") ``` +### User space threads and other oddities + +Some libraries provide user-space threads. These will change the C stack at runtime. +To guard against this we only raise if the stack pointer is in the window between the expected stack base and the soft limit. + ### Diagnosing and fixing stack overflows For stack protection to work correctly the amount of stack consumed between calls to `_Py_EnterRecursiveCall()` must be less than `_PyOS_STACK_MARGIN_BYTES`. diff --git a/InternalDocs/stackrefs.md b/InternalDocs/stackrefs.md new file mode 100644 index 00000000000..5774be9c56d --- /dev/null +++ b/InternalDocs/stackrefs.md @@ -0,0 +1,79 @@ +# Stack references (`_PyStackRef`) + +Stack references are the interpreter's tagged representation of values on the evaluation stack. +They carry metadata to track ownership and support optimizations such as tagged small ints. + +## Shape and tagging + +- A `_PyStackRef` is a tagged pointer-sized value (see `Include/internal/pycore_stackref.h`). +- Tag bits distinguish three cases: + - `Py_TAG_REFCNT` unset - reference count lives on the pointed-to object. + - `Py_TAG_REFCNT` set - ownership is "borrowed" (no refcount to drop on close) or the object is immortal. + - `Py_INT_TAG` set - tagged small integer stored directly in the stackref (no heap allocation). +- Special constants: `PyStackRef_NULL`, `PyStackRef_ERROR`, and embedded `None`/`True`/`False`. + +In GIL builds, most objects carry their refcount; tagged borrowed refs skip decref on close. In free +threading builds, the tag is also used to mark deferred refcounted objects so the GC can see them and +to avoid refcount contention on commonly shared objects. + +## Converting to and from PyObject* + +Three conversions control ownership: + +- `PyStackRef_FromPyObjectNew(obj)` - create a new reference (INCREF if mortal). +- `PyStackRef_FromPyObjectSteal(obj)` - take over ownership without changing the count unless the + object is immortal. +- `PyStackRef_FromPyObjectBorrow(obj)` - create a borrowed stackref (never decref on close). + +The `obj` argument must not be `NULL`. + +Going back to `PyObject*` mirrors this: + +- `PyStackRef_AsPyObjectBorrow(ref)` - borrow the underlying pointer +- `PyStackRef_AsPyObjectSteal(ref)` - transfer ownership from the stackref; if ref is borrowed or + deferred, this creates a new owning `PyObject*` reference. +- `PyStackRef_AsPyObjectNew(ref)` - create a new owning reference + +Only `PyStackRef_AsPyObjectBorrow` allows ref to be `PyStackRef_NULL`. + +## Operations on stackrefs + +The interpreter treats `_PyStackRef` as the unit of stack storage. Ownership must be managed with +the stackref primitives: + +- `PyStackRef_DUP` - like `Py_NewRef` for stackrefs; preserves the original. +- `PyStackRef_Borrow` - create a borrowed stackref from another stackref. +- `PyStackRef_CLOSE` / `PyStackRef_XCLOSE` - like `Py_DECREF`; invalidates the stackref. +- `PyStackRef_CLEAR` - like `Py_CLEAR`; closes and sets the stackref to `PyStackRef_NULL` +- `PyStackRef_MakeHeapSafe` - converts borrowed reference to owning reference + +Borrow tracking (for debug builds with `Py_STACKREF_DEBUG`) records who you borrowed from and reports +double-close, leaked borrows, or use-after-close via fatal errors. + +## Borrow-friendly opcodes + +The interpreter can push borrowed references directly. For example, `LOAD_FAST_BORROW` loads a local +variable as a borrowed `_PyStackRef`, avoiding both INCREF and DECREF for the temporary lifetime on +the evaluation stack. + +## Tagged integers on the stack + +Small ints can be stored inline with `Py_INT_TAG`, so no heap object is involved. Helpers like +`PyStackRef_TagInt`, `PyStackRef_UntagInt`, and `PyStackRef_IncrementTaggedIntNoOverflow` operate on +these values. Type checks use `PyStackRef_IsTaggedInt` and `PyStackRef_LongCheck`. + +## Free threading considerations + +Objects that support deferred reference counting can be pushed to the evaluation +stack and stored in local variables without directly incrementing the reference +count because they are only freed during cyclic garbage collection. This avoids +reference count contention on commonly shared objects such as methods and types. The GC +scans each thread's locals and evaluation stack to keep objects that use +deferred reference counting alive. + +## Debugging support + +`Py_STACKREF_DEBUG` builds replace the inline tags with table-backed IDs so the runtime can track +creation sites, borrows, closes, and leaks. Enabling `Py_STACKREF_CLOSE_DEBUG` additionally records +double closes. The tables live on `PyInterpreterState` and are initialized in `pystate.c`; helper +routines reside in `Python/stackrefs.c`. diff --git a/InternalDocs/string_interning.md b/InternalDocs/string_interning.md index 26a5197c6e7..829a27654a3 100644 --- a/InternalDocs/string_interning.md +++ b/InternalDocs/string_interning.md @@ -16,8 +16,8 @@ ## Singletons The 256 possible one-character latin-1 strings, which can be retrieved with `_Py_LATIN1_CHR(c)`, are stored in statically allocated arrays, -`_PyRuntime.static_objects.strings.ascii` and -`_PyRuntime.static_objects.strings.latin1`. +`_PyRuntime.static_objects.singletons.strings.ascii` and +`_PyRuntime.static_objects.singletons.strings.latin1`. Longer singleton strings are marked in C source with `_Py_ID` (if the string is a valid C identifier fragment) or `_Py_STR` (if it needs a separate @@ -52,15 +52,9 @@ ## Dynamically allocated strings ## Immortality and reference counting -Invariant: Every immortal string is interned. +In the GIL-enabled build interned strings may be mortal or immortal. In the +free-threaded build, interned strings are always immortal. -In practice, this means that you must not use `_Py_SetImmortal` on -a string. (If you know it's already immortal, don't immortalize it; -if you know it's not interned you might be immortalizing a redundant copy; -if it's interned and mortal it needs extra processing in -`_PyUnicode_InternImmortal`.) - -The converse is not true: interned strings can be mortal. For mortal interned strings: - the 2 references from the interned dict (key & value) are excluded from diff --git a/InternalDocs/structure.md b/InternalDocs/structure.md new file mode 100644 index 00000000000..75c8476aa0a --- /dev/null +++ b/InternalDocs/structure.md @@ -0,0 +1,40 @@ +# CPython source code + +This section gives an overview of CPython's code structure and provides +a summary of file locations for modules and built-ins. + + +## Source code layout + +For a Python module, the typical layout is: + +* `Lib/.py` +* `Modules/_.c` (if there's also a C accelerator module) +* `Lib/test/test_.py` +* `Doc/library/.rst` + +For an extension module, the typical layout is: + +* `Modules/module.c` +* `Lib/test/test_.py` +* `Doc/library/.rst` + +For builtin types, the typical layout is: + +* `Objects/object.c` +* `Lib/test/test_.py` +* [`Doc/library/stdtypes.rst`](../Doc/library/stdtypes.rst) + +For builtin functions, the typical layout is: + +* [`Python/bltinmodule.c`](../Python/bltinmodule.c) +* [`Lib/test/test_builtin.py`](../Lib/test/test_builtin.py) +* [`Doc/library/functions.rst`](../Doc/library/functions.rst) + +Some exceptions to these layouts are: + +* built-in type `int` is at [`Objects/longobject.c`](../Objects/longobject.c) +* built-in type `str` is at [`Objects/unicodeobject.c`](../Objects/unicodeobject.c) +* built-in module `sys` is at [`Python/sysmodule.c`](../Python/sysmodule.c) +* built-in module `marshal` is at [`Python/marshal.c`](../Python/marshal.c) +* Windows-only module `winreg` is at [`PC/winreg.c`](../PC/winreg.c) diff --git a/Lib/_android_support.py b/Lib/_android_support.py index ae506f6a4b5..320dab52acd 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -29,15 +29,19 @@ def init_streams(android_log_write, stdout_prio, stderr_prio): global logcat logcat = Logcat(android_log_write) - - sys.stdout = TextLogStream( - stdout_prio, "python.stdout", sys.stdout.fileno()) - sys.stderr = TextLogStream( - stderr_prio, "python.stderr", sys.stderr.fileno()) + sys.stdout = TextLogStream(stdout_prio, "python.stdout", sys.stdout) + sys.stderr = TextLogStream(stderr_prio, "python.stderr", sys.stderr) class TextLogStream(io.TextIOWrapper): - def __init__(self, prio, tag, fileno=None, **kwargs): + def __init__(self, prio, tag, original=None, **kwargs): + # Respect the -u option. + if original: + kwargs.setdefault("write_through", original.write_through) + fileno = original.fileno() + else: + fileno = None + # The default is surrogateescape for stdout and backslashreplace for # stderr, but in the context of an Android log, readability is more # important than reversibility. @@ -164,6 +168,13 @@ def write(self, prio, tag, message): # message. message = message.replace(b"\x00", b"\xc0\x80") + # On API level 30 and higher, Logcat will strip any number of leading + # newlines. This is visible in all `logcat` modes, even --binary. Work + # around this by adding a leading space, which shouldn't make any + # difference to the log's usability. + if message.startswith(b"\n"): + message = b" " + message + with self._lock: now = time() self._bucket_level += ( diff --git a/Lib/_ast_unparse.py b/Lib/_ast_unparse.py index 16cf56f62cc..916bb25d74d 100644 --- a/Lib/_ast_unparse.py +++ b/Lib/_ast_unparse.py @@ -239,11 +239,11 @@ def visit_NamedExpr(self, node): self.traverse(node.value) def visit_Import(self, node): - self.fill("import ") + self.fill("lazy import " if node.is_lazy else "import ") self.interleave(lambda: self.write(", "), self.traverse, node.names) def visit_ImportFrom(self, node): - self.fill("from ") + self.fill("lazy from " if node.is_lazy else "from ") self.write("." * (node.level or 0)) if node.module: self.write(node.module) @@ -658,9 +658,9 @@ def _unparse_interpolation_value(self, inner): unparser.set_precedence(_Precedence.TEST.next(), inner) return unparser.visit(inner) - def _write_interpolation(self, node, is_interpolation=False): + def _write_interpolation(self, node, use_str_attr=False): with self.delimit("{", "}"): - if is_interpolation: + if use_str_attr: expr = node.str else: expr = self._unparse_interpolation_value(node.value) @@ -678,7 +678,8 @@ def visit_FormattedValue(self, node): self._write_interpolation(node) def visit_Interpolation(self, node): - self._write_interpolation(node, is_interpolation=True) + # If `str` is set to `None`, use the `value` to generate the source code. + self._write_interpolation(node, use_str_attr=node.str is not None) def visit_Name(self, node): self.write(node.id) @@ -737,9 +738,13 @@ def visit_SetComp(self, node): def visit_DictComp(self, node): with self.delimit("{", "}"): - self.traverse(node.key) - self.write(": ") - self.traverse(node.value) + if node.value: + self.traverse(node.key) + self.write(": ") + self.traverse(node.value) + else: + self.write("**") + self.traverse(node.key) for gen in node.generators: self.traverse(gen) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 51263d696a1..23cc6d8faae 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -823,6 +823,7 @@ def __eq__(self, other): __reversed__ = None +Mapping.register(frozendict) Mapping.register(mappingproxy) Mapping.register(framelocalsproxy) @@ -1061,6 +1062,41 @@ def count(self, value): Sequence.register(range) Sequence.register(memoryview) +class _DeprecateByteStringMeta(ABCMeta): + def __new__(cls, name, bases, namespace, **kwargs): + if name != "ByteString": + import warnings + + warnings._deprecated( + "collections.abc.ByteString", + remove=(3, 17), + ) + return super().__new__(cls, name, bases, namespace, **kwargs) + + def __instancecheck__(cls, instance): + import warnings + + warnings._deprecated( + "collections.abc.ByteString", + remove=(3, 17), + ) + return super().__instancecheck__(instance) + +class ByteString(Sequence, metaclass=_DeprecateByteStringMeta): + """Deprecated ABC serving as a common supertype of ``bytes`` and ``bytearray``. + + This ABC is scheduled for removal in Python 3.17. + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the buffer protocol at runtime. For use in type annotations, + either use ``Buffer`` or a union that explicitly specifies the types your + code supports (e.g., ``bytes | bytearray | memoryview``). + """ + + __slots__ = () + +ByteString.register(bytes) +ByteString.register(bytearray) + class MutableSequence(Sequence): """All the operations on a read-write sequence. @@ -1130,3 +1166,13 @@ def __iadd__(self, values): MutableSequence.register(list) MutableSequence.register(bytearray) + +_deprecated_ByteString = globals().pop("ByteString") + +def __getattr__(attr): + if attr == "ByteString": + import warnings + warnings._deprecated("collections.abc.ByteString", remove=(3, 17)) + globals()["ByteString"] = _deprecated_ByteString + return _deprecated_ByteString + raise AttributeError(f"module 'collections.abc' has no attribute {attr!r}") diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 325efed274a..62806b1d8d7 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -1,4 +1,4 @@ -import io +import builtins import os import sys @@ -10,13 +10,12 @@ # types if False: - from typing import IO, Self, ClassVar + from typing import IO, Literal, Self, ClassVar _theme: Theme class ANSIColors: RESET = "\x1b[0m" - BLACK = "\x1b[30m" BLUE = "\x1b[34m" CYAN = "\x1b[36m" @@ -75,6 +74,19 @@ class ANSIColors: setattr(NoColors, attr, "") +class CursesColors: + """Curses color constants for terminal UI theming.""" + BLACK = 0 + RED = 1 + GREEN = 2 + YELLOW = 3 + BLUE = 4 + MAGENTA = 5 + CYAN = 6 + WHITE = 7 + DEFAULT = -1 + + # # Experimental theming support (see gh-133346) # @@ -155,7 +167,7 @@ def __iter__(self) -> Iterator[str]: return iter(self.__dataclass_fields__) -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Argparse(ThemeSection): usage: str = ANSIColors.BOLD_BLUE prog: str = ANSIColors.BOLD_MAGENTA @@ -169,6 +181,22 @@ class Argparse(ThemeSection): short_option: str = ANSIColors.BOLD_GREEN label: str = ANSIColors.BOLD_YELLOW action: str = ANSIColors.BOLD_GREEN + default: str = ANSIColors.GREY + interpolated_value: str = ANSIColors.YELLOW + reset: str = ANSIColors.RESET + error: str = ANSIColors.BOLD_MAGENTA + warning: str = ANSIColors.BOLD_YELLOW + message: str = ANSIColors.MAGENTA + + +@dataclass(frozen=True, kw_only=True) +class Ast(ThemeSection): + node: str = ANSIColors.CYAN + field: str = ANSIColors.BLUE + attribute: str = ANSIColors.GREY + string: str = ANSIColors.GREEN + number: str = ANSIColors.YELLOW + keyword: str = ANSIColors.BOLD_BLUE reset: str = ANSIColors.RESET @@ -183,10 +211,176 @@ class Difflib(ThemeSection): reset: str = ANSIColors.RESET +@dataclass(frozen=True, kw_only=True) +class FancyCompleter(ThemeSection): + # functions and methods + function: builtins.str = ANSIColors.BOLD_BLUE + builtin_function_or_method: builtins.str = ANSIColors.BOLD_BLUE + method: builtins.str = ANSIColors.BOLD_CYAN + method_wrapper: builtins.str = ANSIColors.BOLD_CYAN + wrapper_descriptor: builtins.str = ANSIColors.BOLD_CYAN + method_descriptor: builtins.str = ANSIColors.BOLD_CYAN + + # numbers + int: builtins.str = ANSIColors.BOLD_YELLOW + float: builtins.str = ANSIColors.BOLD_YELLOW + complex: builtins.str = ANSIColors.BOLD_YELLOW + bool: builtins.str = ANSIColors.BOLD_YELLOW + + # others + type: builtins.str = ANSIColors.BOLD_MAGENTA + module: builtins.str = ANSIColors.CYAN + NoneType: builtins.str = ANSIColors.GREY + bytes: builtins.str = ANSIColors.BOLD_GREEN + str: builtins.str = ANSIColors.BOLD_GREEN + + +@dataclass(frozen=True, kw_only=True) +class HttpServer(ThemeSection): + error: str = ANSIColors.YELLOW + path: str = ANSIColors.CYAN + serving: str = ANSIColors.GREEN + size: str = ANSIColors.GREY + status_informational: str = ANSIColors.RESET + status_ok: str = ANSIColors.GREEN + status_redirect: str = ANSIColors.INTENSE_CYAN + status_client_error: str = ANSIColors.YELLOW + status_server_error: str = ANSIColors.RED + timestamp: str = ANSIColors.GREY + url: str = ANSIColors.CYAN + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class LiveProfiler(ThemeSection): + """Theme section for the live profiling TUI (Tachyon profiler). + + Colors use CursesColors constants (BLACK, RED, GREEN, YELLOW, + BLUE, MAGENTA, CYAN, WHITE, DEFAULT). + """ + # Header colors + title_fg: int = CursesColors.CYAN + title_bg: int = CursesColors.DEFAULT + + # Status display colors + pid_fg: int = CursesColors.CYAN + uptime_fg: int = CursesColors.GREEN + time_fg: int = CursesColors.YELLOW + interval_fg: int = CursesColors.MAGENTA + + # Thread view colors + thread_all_fg: int = CursesColors.GREEN + thread_single_fg: int = CursesColors.MAGENTA + + # Progress bar colors + bar_good_fg: int = CursesColors.GREEN + bar_bad_fg: int = CursesColors.RED + + # Stats colors + on_gil_fg: int = CursesColors.GREEN + off_gil_fg: int = CursesColors.RED + waiting_gil_fg: int = CursesColors.YELLOW + gc_fg: int = CursesColors.MAGENTA + + # Function display colors + func_total_fg: int = CursesColors.CYAN + func_exec_fg: int = CursesColors.GREEN + func_stack_fg: int = CursesColors.YELLOW + func_shown_fg: int = CursesColors.MAGENTA + + # Table header colors (for sorted column highlight) + sorted_header_fg: int = CursesColors.BLACK + sorted_header_bg: int = CursesColors.CYAN + + # Normal header colors (non-sorted columns) - use reverse video style + normal_header_fg: int = CursesColors.BLACK + normal_header_bg: int = CursesColors.WHITE + + # Data row colors + samples_fg: int = CursesColors.CYAN + file_fg: int = CursesColors.GREEN + func_fg: int = CursesColors.YELLOW + + # Trend indicator colors + trend_up_fg: int = CursesColors.GREEN + trend_down_fg: int = CursesColors.RED + + # Medal colors for top functions + medal_gold_fg: int = CursesColors.RED + medal_silver_fg: int = CursesColors.YELLOW + medal_bronze_fg: int = CursesColors.GREEN + + # Background style: 'dark' or 'light' + background_style: Literal["dark", "light"] = "dark" + + +LiveProfilerLight = LiveProfiler( + # Header colors + title_fg=CursesColors.BLUE, # Blue is more readable than cyan on light bg + + # Status display colors - darker colors for light backgrounds + pid_fg=CursesColors.BLUE, + uptime_fg=CursesColors.BLACK, + time_fg=CursesColors.BLACK, + interval_fg=CursesColors.BLUE, + + # Thread view colors + thread_all_fg=CursesColors.BLACK, + thread_single_fg=CursesColors.BLUE, + + # Stats colors + waiting_gil_fg=CursesColors.RED, + gc_fg=CursesColors.BLUE, + + # Function display colors + func_total_fg=CursesColors.BLUE, + func_exec_fg=CursesColors.BLACK, + func_stack_fg=CursesColors.BLACK, + func_shown_fg=CursesColors.BLUE, + + # Table header colors (for sorted column highlight) + sorted_header_fg=CursesColors.WHITE, + sorted_header_bg=CursesColors.BLUE, + + # Normal header colors (non-sorted columns) + normal_header_fg=CursesColors.WHITE, + normal_header_bg=CursesColors.BLACK, + + # Data row colors - use dark colors readable on white + samples_fg=CursesColors.BLACK, + file_fg=CursesColors.BLACK, + func_fg=CursesColors.BLUE, # Blue is more readable than magenta on light bg + + # Medal colors for top functions + medal_silver_fg=CursesColors.BLUE, + + # Background style + background_style="light", +) + + +@dataclass(frozen=True, kw_only=True) +class Pickletools(ThemeSection): + annotation: str = ANSIColors.GREY + arg_number: str = ANSIColors.YELLOW + arg_string: str = ANSIColors.GREEN + mark: str = ANSIColors.GREY + op_call: str = ANSIColors.GREEN + op_container: str = ANSIColors.INTENSE_BLUE + op_memo: str = ANSIColors.MAGENTA + op_meta: str = ANSIColors.GREY + op_stack: str = ANSIColors.BOLD_RED + opcode_code: str = ANSIColors.CYAN + position: str = ANSIColors.GREY + proto: str = ANSIColors.YELLOW + reset: str = ANSIColors.RESET + + @dataclass(frozen=True, kw_only=True) class Syntax(ThemeSection): prompt: str = ANSIColors.BOLD_MAGENTA keyword: str = ANSIColors.BOLD_BLUE + keyword_constant: str = ANSIColors.BOLD_BLUE builtin: str = ANSIColors.CYAN comment: str = ANSIColors.RED string: str = ANSIColors.GREEN @@ -197,10 +391,31 @@ class Syntax(ThemeSection): reset: str = ANSIColors.RESET +@dataclass(frozen=True, kw_only=True) +class Timeit(ThemeSection): + timing: str = ANSIColors.CYAN + best: str = ANSIColors.BOLD_GREEN + per_loop: str = ANSIColors.GREEN + punctuation: str = ANSIColors.GREY + warning: str = ANSIColors.YELLOW + warning_worst: str = ANSIColors.MAGENTA + warning_best: str = ANSIColors.GREEN + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class Tokenize(ThemeSection): + whitespace: str = ANSIColors.GREY + error: str = ANSIColors.BOLD_RED + position: str = ANSIColors.GREY + delimiter: str = ANSIColors.RESET + + @dataclass(frozen=True, kw_only=True) class Traceback(ThemeSection): type: str = ANSIColors.BOLD_MAGENTA message: str = ANSIColors.MAGENTA + note: str = ANSIColors.CYAN filename: str = ANSIColors.MAGENTA line_no: str = ANSIColors.MAGENTA frame: str = ANSIColors.MAGENTA @@ -226,8 +441,15 @@ class Theme: below. """ argparse: Argparse = field(default_factory=Argparse) + ast: Ast = field(default_factory=Ast) difflib: Difflib = field(default_factory=Difflib) + fancycompleter: FancyCompleter = field(default_factory=FancyCompleter) + http_server: HttpServer = field(default_factory=HttpServer) + live_profiler: LiveProfiler = field(default_factory=LiveProfiler) + pickletools: Pickletools = field(default_factory=Pickletools) syntax: Syntax = field(default_factory=Syntax) + timeit: Timeit = field(default_factory=Timeit) + tokenize: Tokenize = field(default_factory=Tokenize) traceback: Traceback = field(default_factory=Traceback) unittest: Unittest = field(default_factory=Unittest) @@ -235,8 +457,15 @@ def copy_with( self, *, argparse: Argparse | None = None, + ast: Ast | None = None, difflib: Difflib | None = None, + fancycompleter: FancyCompleter | None = None, + http_server: HttpServer | None = None, + live_profiler: LiveProfiler | None = None, + pickletools: Pickletools | None = None, syntax: Syntax | None = None, + timeit: Timeit | None = None, + tokenize: Tokenize | None = None, traceback: Traceback | None = None, unittest: Unittest | None = None, ) -> Self: @@ -247,8 +476,15 @@ def copy_with( """ return type(self)( argparse=argparse or self.argparse, + ast=ast or self.ast, difflib=difflib or self.difflib, + fancycompleter=fancycompleter or self.fancycompleter, + http_server=http_server or self.http_server, + live_profiler=live_profiler or self.live_profiler, + pickletools=pickletools or self.pickletools, syntax=syntax or self.syntax, + timeit=timeit or self.timeit, + tokenize=tokenize or self.tokenize, traceback=traceback or self.traceback, unittest=unittest or self.unittest, ) @@ -263,8 +499,15 @@ def no_colors(cls) -> Self: """ return cls( argparse=Argparse.no_colors(), + ast=Ast.no_colors(), difflib=Difflib.no_colors(), + fancycompleter=FancyCompleter.no_colors(), + http_server=HttpServer.no_colors(), + live_profiler=LiveProfiler.no_colors(), + pickletools=Pickletools.no_colors(), syntax=Syntax.no_colors(), + timeit=Timeit.no_colors(), + tokenize=Tokenize.no_colors(), traceback=Traceback.no_colors(), unittest=Unittest.no_colors(), ) @@ -287,21 +530,29 @@ def decolor(text: str) -> str: def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool: + + def _safe_getenv(k: str, fallback: str | None = None) -> str | None: + """Exception-safe environment retrieval. See gh-128636.""" + try: + return os.environ.get(k, fallback) + except Exception: + return fallback + if file is None: file = sys.stdout if not sys.flags.ignore_environment: - if os.environ.get("PYTHON_COLORS") == "0": + if _safe_getenv("PYTHON_COLORS") == "0": return False - if os.environ.get("PYTHON_COLORS") == "1": + if _safe_getenv("PYTHON_COLORS") == "1": return True - if os.environ.get("NO_COLOR"): + if _safe_getenv("NO_COLOR"): return False if not COLORIZE: return False - if os.environ.get("FORCE_COLOR"): + if _safe_getenv("FORCE_COLOR"): return True - if os.environ.get("TERM") == "dumb": + if _safe_getenv("TERM") == "dumb": return False if not hasattr(file, "fileno"): @@ -318,13 +569,16 @@ def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool: try: return os.isatty(file.fileno()) - except io.UnsupportedOperation: + except OSError: return hasattr(file, "isatty") and file.isatty() default_theme = Theme() theme_no_color = default_theme.no_colors() +# Convenience theme with light profiler colors (for white/light terminal backgrounds) +light_profiler_theme = default_theme.copy_with(live_profiler=LiveProfilerLight) + def get_theme( *, @@ -344,7 +598,8 @@ def get_theme( environment (including environment variable state and console configuration on Windows) can also change in the course of the application life cycle. """ - if force_color or (not force_no_color and can_colorize(file=tty_file)): + if force_color or (not force_no_color and + can_colorize(file=tty_file)): return _theme return theme_no_color diff --git a/Lib/_compat_pickle.py b/Lib/_compat_pickle.py index a9813264324..928db663b44 100644 --- a/Lib/_compat_pickle.py +++ b/Lib/_compat_pickle.py @@ -240,6 +240,7 @@ REVERSE_NAME_MAPPING[('builtins', excname)] = ('exceptions', 'OSError') PYTHON3_IMPORTERROR_EXCEPTIONS = ( + 'ImportCycleError', 'ModuleNotFoundError', ) diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index f168d169a32..6b5357a3151 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -2,364 +2,382 @@ # from: # Python/bytecodes.c # Do not edit! -_specializations = { - "RESUME": [ - "RESUME_CHECK", - ], - "TO_BOOL": [ - "TO_BOOL_ALWAYS_TRUE", - "TO_BOOL_BOOL", - "TO_BOOL_INT", - "TO_BOOL_LIST", - "TO_BOOL_NONE", - "TO_BOOL_STR", - ], - "BINARY_OP": [ - "BINARY_OP_MULTIPLY_INT", - "BINARY_OP_ADD_INT", - "BINARY_OP_SUBTRACT_INT", - "BINARY_OP_MULTIPLY_FLOAT", - "BINARY_OP_ADD_FLOAT", - "BINARY_OP_SUBTRACT_FLOAT", - "BINARY_OP_ADD_UNICODE", - "BINARY_OP_SUBSCR_LIST_INT", - "BINARY_OP_SUBSCR_LIST_SLICE", - "BINARY_OP_SUBSCR_TUPLE_INT", - "BINARY_OP_SUBSCR_STR_INT", - "BINARY_OP_SUBSCR_DICT", - "BINARY_OP_SUBSCR_GETITEM", - "BINARY_OP_EXTEND", - "BINARY_OP_INPLACE_ADD_UNICODE", - ], - "STORE_SUBSCR": [ - "STORE_SUBSCR_DICT", - "STORE_SUBSCR_LIST_INT", - ], - "SEND": [ - "SEND_GEN", - ], - "UNPACK_SEQUENCE": [ - "UNPACK_SEQUENCE_TWO_TUPLE", - "UNPACK_SEQUENCE_TUPLE", - "UNPACK_SEQUENCE_LIST", - ], - "STORE_ATTR": [ - "STORE_ATTR_INSTANCE_VALUE", - "STORE_ATTR_SLOT", - "STORE_ATTR_WITH_HINT", - ], - "LOAD_GLOBAL": [ - "LOAD_GLOBAL_MODULE", - "LOAD_GLOBAL_BUILTIN", - ], - "LOAD_SUPER_ATTR": [ - "LOAD_SUPER_ATTR_ATTR", - "LOAD_SUPER_ATTR_METHOD", - ], - "LOAD_ATTR": [ - "LOAD_ATTR_INSTANCE_VALUE", - "LOAD_ATTR_MODULE", - "LOAD_ATTR_WITH_HINT", - "LOAD_ATTR_SLOT", - "LOAD_ATTR_CLASS", - "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK", - "LOAD_ATTR_PROPERTY", - "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - "LOAD_ATTR_METHOD_WITH_VALUES", - "LOAD_ATTR_METHOD_NO_DICT", - "LOAD_ATTR_METHOD_LAZY_DICT", - "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", - "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", - ], - "COMPARE_OP": [ - "COMPARE_OP_FLOAT", - "COMPARE_OP_INT", - "COMPARE_OP_STR", - ], - "CONTAINS_OP": [ - "CONTAINS_OP_SET", - "CONTAINS_OP_DICT", - ], - "JUMP_BACKWARD": [ - "JUMP_BACKWARD_NO_JIT", - "JUMP_BACKWARD_JIT", - ], - "FOR_ITER": [ - "FOR_ITER_LIST", - "FOR_ITER_TUPLE", - "FOR_ITER_RANGE", - "FOR_ITER_GEN", - ], - "CALL": [ - "CALL_BOUND_METHOD_EXACT_ARGS", - "CALL_PY_EXACT_ARGS", - "CALL_TYPE_1", - "CALL_STR_1", - "CALL_TUPLE_1", - "CALL_BUILTIN_CLASS", - "CALL_BUILTIN_O", - "CALL_BUILTIN_FAST", - "CALL_BUILTIN_FAST_WITH_KEYWORDS", - "CALL_LEN", - "CALL_ISINSTANCE", - "CALL_LIST_APPEND", - "CALL_METHOD_DESCRIPTOR_O", - "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - "CALL_METHOD_DESCRIPTOR_NOARGS", - "CALL_METHOD_DESCRIPTOR_FAST", - "CALL_ALLOC_AND_ENTER_INIT", - "CALL_PY_GENERAL", - "CALL_BOUND_METHOD_GENERAL", - "CALL_NON_PY_GENERAL", - ], - "CALL_KW": [ - "CALL_KW_BOUND_METHOD", - "CALL_KW_PY", - "CALL_KW_NON_PY", - ], -} +_specializations = frozendict( + RESUME=( + "RESUME_CHECK", + "RESUME_CHECK_JIT", + ), + TO_BOOL=( + "TO_BOOL_ALWAYS_TRUE", + "TO_BOOL_BOOL", + "TO_BOOL_INT", + "TO_BOOL_LIST", + "TO_BOOL_NONE", + "TO_BOOL_STR", + ), + BINARY_OP=( + "BINARY_OP_MULTIPLY_INT", + "BINARY_OP_ADD_INT", + "BINARY_OP_SUBTRACT_INT", + "BINARY_OP_MULTIPLY_FLOAT", + "BINARY_OP_ADD_FLOAT", + "BINARY_OP_SUBTRACT_FLOAT", + "BINARY_OP_ADD_UNICODE", + "BINARY_OP_SUBSCR_LIST_INT", + "BINARY_OP_SUBSCR_LIST_SLICE", + "BINARY_OP_SUBSCR_TUPLE_INT", + "BINARY_OP_SUBSCR_STR_INT", + "BINARY_OP_SUBSCR_USTR_INT", + "BINARY_OP_SUBSCR_DICT", + "BINARY_OP_SUBSCR_GETITEM", + "BINARY_OP_INPLACE_ADD_UNICODE", + "BINARY_OP_EXTEND", + ), + STORE_SUBSCR=( + "STORE_SUBSCR_DICT", + "STORE_SUBSCR_LIST_INT", + ), + SEND=( + "SEND_GEN", + ), + UNPACK_SEQUENCE=( + "UNPACK_SEQUENCE_TWO_TUPLE", + "UNPACK_SEQUENCE_TUPLE", + "UNPACK_SEQUENCE_LIST", + ), + STORE_ATTR=( + "STORE_ATTR_INSTANCE_VALUE", + "STORE_ATTR_SLOT", + "STORE_ATTR_WITH_HINT", + ), + LOAD_GLOBAL=( + "LOAD_GLOBAL_MODULE", + "LOAD_GLOBAL_BUILTIN", + ), + LOAD_SUPER_ATTR=( + "LOAD_SUPER_ATTR_ATTR", + "LOAD_SUPER_ATTR_METHOD", + ), + LOAD_ATTR=( + "LOAD_ATTR_INSTANCE_VALUE", + "LOAD_ATTR_MODULE", + "LOAD_ATTR_WITH_HINT", + "LOAD_ATTR_SLOT", + "LOAD_ATTR_CLASS", + "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK", + "LOAD_ATTR_PROPERTY", + "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + "LOAD_ATTR_METHOD_WITH_VALUES", + "LOAD_ATTR_METHOD_NO_DICT", + "LOAD_ATTR_METHOD_LAZY_DICT", + "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + ), + COMPARE_OP=( + "COMPARE_OP_FLOAT", + "COMPARE_OP_INT", + "COMPARE_OP_STR", + ), + CONTAINS_OP=( + "CONTAINS_OP_SET", + "CONTAINS_OP_DICT", + ), + JUMP_BACKWARD=( + "JUMP_BACKWARD_NO_JIT", + "JUMP_BACKWARD_JIT", + ), + GET_ITER=( + "GET_ITER_SELF", + "GET_ITER_VIRTUAL", + ), + FOR_ITER=( + "FOR_ITER_LIST", + "FOR_ITER_TUPLE", + "FOR_ITER_RANGE", + "FOR_ITER_GEN", + "FOR_ITER_VIRTUAL", + ), + CALL=( + "CALL_BOUND_METHOD_EXACT_ARGS", + "CALL_PY_EXACT_ARGS", + "CALL_TYPE_1", + "CALL_STR_1", + "CALL_TUPLE_1", + "CALL_BUILTIN_CLASS", + "CALL_BUILTIN_O", + "CALL_BUILTIN_FAST", + "CALL_BUILTIN_FAST_WITH_KEYWORDS", + "CALL_LEN", + "CALL_ISINSTANCE", + "CALL_LIST_APPEND", + "CALL_METHOD_DESCRIPTOR_O", + "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + "CALL_METHOD_DESCRIPTOR_NOARGS", + "CALL_METHOD_DESCRIPTOR_FAST", + "CALL_ALLOC_AND_ENTER_INIT", + "CALL_PY_GENERAL", + "CALL_BOUND_METHOD_GENERAL", + "CALL_NON_PY_GENERAL", + ), + CALL_KW=( + "CALL_KW_BOUND_METHOD", + "CALL_KW_PY", + "CALL_KW_NON_PY", + ), + CALL_FUNCTION_EX=( + "CALL_EX_PY", + "CALL_EX_NON_PY_GENERAL", + ), +) -_specialized_opmap = { - 'BINARY_OP_ADD_FLOAT': 129, - 'BINARY_OP_ADD_INT': 130, - 'BINARY_OP_ADD_UNICODE': 131, - 'BINARY_OP_EXTEND': 132, - 'BINARY_OP_INPLACE_ADD_UNICODE': 3, - 'BINARY_OP_MULTIPLY_FLOAT': 133, - 'BINARY_OP_MULTIPLY_INT': 134, - 'BINARY_OP_SUBSCR_DICT': 135, - 'BINARY_OP_SUBSCR_GETITEM': 136, - 'BINARY_OP_SUBSCR_LIST_INT': 137, - 'BINARY_OP_SUBSCR_LIST_SLICE': 138, - 'BINARY_OP_SUBSCR_STR_INT': 139, - 'BINARY_OP_SUBSCR_TUPLE_INT': 140, - 'BINARY_OP_SUBTRACT_FLOAT': 141, - 'BINARY_OP_SUBTRACT_INT': 142, - 'CALL_ALLOC_AND_ENTER_INIT': 143, - 'CALL_BOUND_METHOD_EXACT_ARGS': 144, - 'CALL_BOUND_METHOD_GENERAL': 145, - 'CALL_BUILTIN_CLASS': 146, - 'CALL_BUILTIN_FAST': 147, - 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 148, - 'CALL_BUILTIN_O': 149, - 'CALL_ISINSTANCE': 150, - 'CALL_KW_BOUND_METHOD': 151, - 'CALL_KW_NON_PY': 152, - 'CALL_KW_PY': 153, - 'CALL_LEN': 154, - 'CALL_LIST_APPEND': 155, - 'CALL_METHOD_DESCRIPTOR_FAST': 156, - 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 157, - 'CALL_METHOD_DESCRIPTOR_NOARGS': 158, - 'CALL_METHOD_DESCRIPTOR_O': 159, - 'CALL_NON_PY_GENERAL': 160, - 'CALL_PY_EXACT_ARGS': 161, - 'CALL_PY_GENERAL': 162, - 'CALL_STR_1': 163, - 'CALL_TUPLE_1': 164, - 'CALL_TYPE_1': 165, - 'COMPARE_OP_FLOAT': 166, - 'COMPARE_OP_INT': 167, - 'COMPARE_OP_STR': 168, - 'CONTAINS_OP_DICT': 169, - 'CONTAINS_OP_SET': 170, - 'FOR_ITER_GEN': 171, - 'FOR_ITER_LIST': 172, - 'FOR_ITER_RANGE': 173, - 'FOR_ITER_TUPLE': 174, - 'JUMP_BACKWARD_JIT': 175, - 'JUMP_BACKWARD_NO_JIT': 176, - 'LOAD_ATTR_CLASS': 177, - 'LOAD_ATTR_CLASS_WITH_METACLASS_CHECK': 178, - 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 179, - 'LOAD_ATTR_INSTANCE_VALUE': 180, - 'LOAD_ATTR_METHOD_LAZY_DICT': 181, - 'LOAD_ATTR_METHOD_NO_DICT': 182, - 'LOAD_ATTR_METHOD_WITH_VALUES': 183, - 'LOAD_ATTR_MODULE': 184, - 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 185, - 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 186, - 'LOAD_ATTR_PROPERTY': 187, - 'LOAD_ATTR_SLOT': 188, - 'LOAD_ATTR_WITH_HINT': 189, - 'LOAD_GLOBAL_BUILTIN': 190, - 'LOAD_GLOBAL_MODULE': 191, - 'LOAD_SUPER_ATTR_ATTR': 192, - 'LOAD_SUPER_ATTR_METHOD': 193, - 'RESUME_CHECK': 194, - 'SEND_GEN': 195, - 'STORE_ATTR_INSTANCE_VALUE': 196, - 'STORE_ATTR_SLOT': 197, - 'STORE_ATTR_WITH_HINT': 198, - 'STORE_SUBSCR_DICT': 199, - 'STORE_SUBSCR_LIST_INT': 200, - 'TO_BOOL_ALWAYS_TRUE': 201, - 'TO_BOOL_BOOL': 202, - 'TO_BOOL_INT': 203, - 'TO_BOOL_LIST': 204, - 'TO_BOOL_NONE': 205, - 'TO_BOOL_STR': 206, - 'UNPACK_SEQUENCE_LIST': 207, - 'UNPACK_SEQUENCE_TUPLE': 208, - 'UNPACK_SEQUENCE_TWO_TUPLE': 209, -} +_specialized_opmap = frozendict( + BINARY_OP_ADD_FLOAT=129, + BINARY_OP_ADD_INT=130, + BINARY_OP_ADD_UNICODE=131, + BINARY_OP_EXTEND=132, + BINARY_OP_INPLACE_ADD_UNICODE=3, + BINARY_OP_MULTIPLY_FLOAT=133, + BINARY_OP_MULTIPLY_INT=134, + BINARY_OP_SUBSCR_DICT=135, + BINARY_OP_SUBSCR_GETITEM=136, + BINARY_OP_SUBSCR_LIST_INT=137, + BINARY_OP_SUBSCR_LIST_SLICE=138, + BINARY_OP_SUBSCR_STR_INT=139, + BINARY_OP_SUBSCR_TUPLE_INT=140, + BINARY_OP_SUBSCR_USTR_INT=141, + BINARY_OP_SUBTRACT_FLOAT=142, + BINARY_OP_SUBTRACT_INT=143, + CALL_ALLOC_AND_ENTER_INIT=144, + CALL_BOUND_METHOD_EXACT_ARGS=145, + CALL_BOUND_METHOD_GENERAL=146, + CALL_BUILTIN_CLASS=147, + CALL_BUILTIN_FAST=148, + CALL_BUILTIN_FAST_WITH_KEYWORDS=149, + CALL_BUILTIN_O=150, + CALL_EX_NON_PY_GENERAL=151, + CALL_EX_PY=152, + CALL_ISINSTANCE=153, + CALL_KW_BOUND_METHOD=154, + CALL_KW_NON_PY=155, + CALL_KW_PY=156, + CALL_LEN=157, + CALL_LIST_APPEND=158, + CALL_METHOD_DESCRIPTOR_FAST=159, + CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS=160, + CALL_METHOD_DESCRIPTOR_NOARGS=161, + CALL_METHOD_DESCRIPTOR_O=162, + CALL_NON_PY_GENERAL=163, + CALL_PY_EXACT_ARGS=164, + CALL_PY_GENERAL=165, + CALL_STR_1=166, + CALL_TUPLE_1=167, + CALL_TYPE_1=168, + COMPARE_OP_FLOAT=169, + COMPARE_OP_INT=170, + COMPARE_OP_STR=171, + CONTAINS_OP_DICT=172, + CONTAINS_OP_SET=173, + FOR_ITER_GEN=174, + FOR_ITER_LIST=175, + FOR_ITER_RANGE=176, + FOR_ITER_TUPLE=177, + FOR_ITER_VIRTUAL=178, + GET_ITER_SELF=179, + GET_ITER_VIRTUAL=180, + JUMP_BACKWARD_JIT=181, + JUMP_BACKWARD_NO_JIT=182, + LOAD_ATTR_CLASS=183, + LOAD_ATTR_CLASS_WITH_METACLASS_CHECK=184, + LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN=185, + LOAD_ATTR_INSTANCE_VALUE=186, + LOAD_ATTR_METHOD_LAZY_DICT=187, + LOAD_ATTR_METHOD_NO_DICT=188, + LOAD_ATTR_METHOD_WITH_VALUES=189, + LOAD_ATTR_MODULE=190, + LOAD_ATTR_NONDESCRIPTOR_NO_DICT=191, + LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES=192, + LOAD_ATTR_PROPERTY=193, + LOAD_ATTR_SLOT=194, + LOAD_ATTR_WITH_HINT=195, + LOAD_GLOBAL_BUILTIN=196, + LOAD_GLOBAL_MODULE=197, + LOAD_SUPER_ATTR_ATTR=198, + LOAD_SUPER_ATTR_METHOD=199, + RESUME_CHECK=200, + RESUME_CHECK_JIT=201, + SEND_GEN=202, + STORE_ATTR_INSTANCE_VALUE=203, + STORE_ATTR_SLOT=204, + STORE_ATTR_WITH_HINT=205, + STORE_SUBSCR_DICT=206, + STORE_SUBSCR_LIST_INT=207, + TO_BOOL_ALWAYS_TRUE=208, + TO_BOOL_BOOL=209, + TO_BOOL_INT=210, + TO_BOOL_LIST=211, + TO_BOOL_NONE=212, + TO_BOOL_STR=213, + UNPACK_SEQUENCE_LIST=214, + UNPACK_SEQUENCE_TUPLE=215, + UNPACK_SEQUENCE_TWO_TUPLE=216, +) -opmap = { - 'CACHE': 0, - 'RESERVED': 17, - 'RESUME': 128, - 'INSTRUMENTED_LINE': 254, - 'ENTER_EXECUTOR': 255, - 'BINARY_SLICE': 1, - 'BUILD_TEMPLATE': 2, - 'CALL_FUNCTION_EX': 4, - 'CHECK_EG_MATCH': 5, - 'CHECK_EXC_MATCH': 6, - 'CLEANUP_THROW': 7, - 'DELETE_SUBSCR': 8, - 'END_FOR': 9, - 'END_SEND': 10, - 'EXIT_INIT_CHECK': 11, - 'FORMAT_SIMPLE': 12, - 'FORMAT_WITH_SPEC': 13, - 'GET_AITER': 14, - 'GET_ANEXT': 15, - 'GET_ITER': 16, - 'GET_LEN': 18, - 'GET_YIELD_FROM_ITER': 19, - 'INTERPRETER_EXIT': 20, - 'LOAD_BUILD_CLASS': 21, - 'LOAD_LOCALS': 22, - 'MAKE_FUNCTION': 23, - 'MATCH_KEYS': 24, - 'MATCH_MAPPING': 25, - 'MATCH_SEQUENCE': 26, - 'NOP': 27, - 'NOT_TAKEN': 28, - 'POP_EXCEPT': 29, - 'POP_ITER': 30, - 'POP_TOP': 31, - 'PUSH_EXC_INFO': 32, - 'PUSH_NULL': 33, - 'RETURN_GENERATOR': 34, - 'RETURN_VALUE': 35, - 'SETUP_ANNOTATIONS': 36, - 'STORE_SLICE': 37, - 'STORE_SUBSCR': 38, - 'TO_BOOL': 39, - 'UNARY_INVERT': 40, - 'UNARY_NEGATIVE': 41, - 'UNARY_NOT': 42, - 'WITH_EXCEPT_START': 43, - 'BINARY_OP': 44, - 'BUILD_INTERPOLATION': 45, - 'BUILD_LIST': 46, - 'BUILD_MAP': 47, - 'BUILD_SET': 48, - 'BUILD_SLICE': 49, - 'BUILD_STRING': 50, - 'BUILD_TUPLE': 51, - 'CALL': 52, - 'CALL_INTRINSIC_1': 53, - 'CALL_INTRINSIC_2': 54, - 'CALL_KW': 55, - 'COMPARE_OP': 56, - 'CONTAINS_OP': 57, - 'CONVERT_VALUE': 58, - 'COPY': 59, - 'COPY_FREE_VARS': 60, - 'DELETE_ATTR': 61, - 'DELETE_DEREF': 62, - 'DELETE_FAST': 63, - 'DELETE_GLOBAL': 64, - 'DELETE_NAME': 65, - 'DICT_MERGE': 66, - 'DICT_UPDATE': 67, - 'END_ASYNC_FOR': 68, - 'EXTENDED_ARG': 69, - 'FOR_ITER': 70, - 'GET_AWAITABLE': 71, - 'IMPORT_FROM': 72, - 'IMPORT_NAME': 73, - 'IS_OP': 74, - 'JUMP_BACKWARD': 75, - 'JUMP_BACKWARD_NO_INTERRUPT': 76, - 'JUMP_FORWARD': 77, - 'LIST_APPEND': 78, - 'LIST_EXTEND': 79, - 'LOAD_ATTR': 80, - 'LOAD_COMMON_CONSTANT': 81, - 'LOAD_CONST': 82, - 'LOAD_DEREF': 83, - 'LOAD_FAST': 84, - 'LOAD_FAST_AND_CLEAR': 85, - 'LOAD_FAST_BORROW': 86, - 'LOAD_FAST_BORROW_LOAD_FAST_BORROW': 87, - 'LOAD_FAST_CHECK': 88, - 'LOAD_FAST_LOAD_FAST': 89, - 'LOAD_FROM_DICT_OR_DEREF': 90, - 'LOAD_FROM_DICT_OR_GLOBALS': 91, - 'LOAD_GLOBAL': 92, - 'LOAD_NAME': 93, - 'LOAD_SMALL_INT': 94, - 'LOAD_SPECIAL': 95, - 'LOAD_SUPER_ATTR': 96, - 'MAKE_CELL': 97, - 'MAP_ADD': 98, - 'MATCH_CLASS': 99, - 'POP_JUMP_IF_FALSE': 100, - 'POP_JUMP_IF_NONE': 101, - 'POP_JUMP_IF_NOT_NONE': 102, - 'POP_JUMP_IF_TRUE': 103, - 'RAISE_VARARGS': 104, - 'RERAISE': 105, - 'SEND': 106, - 'SET_ADD': 107, - 'SET_FUNCTION_ATTRIBUTE': 108, - 'SET_UPDATE': 109, - 'STORE_ATTR': 110, - 'STORE_DEREF': 111, - 'STORE_FAST': 112, - 'STORE_FAST_LOAD_FAST': 113, - 'STORE_FAST_STORE_FAST': 114, - 'STORE_GLOBAL': 115, - 'STORE_NAME': 116, - 'SWAP': 117, - 'UNPACK_EX': 118, - 'UNPACK_SEQUENCE': 119, - 'YIELD_VALUE': 120, - 'INSTRUMENTED_END_FOR': 234, - 'INSTRUMENTED_POP_ITER': 235, - 'INSTRUMENTED_END_SEND': 236, - 'INSTRUMENTED_FOR_ITER': 237, - 'INSTRUMENTED_INSTRUCTION': 238, - 'INSTRUMENTED_JUMP_FORWARD': 239, - 'INSTRUMENTED_NOT_TAKEN': 240, - 'INSTRUMENTED_POP_JUMP_IF_TRUE': 241, - 'INSTRUMENTED_POP_JUMP_IF_FALSE': 242, - 'INSTRUMENTED_POP_JUMP_IF_NONE': 243, - 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 244, - 'INSTRUMENTED_RESUME': 245, - 'INSTRUMENTED_RETURN_VALUE': 246, - 'INSTRUMENTED_YIELD_VALUE': 247, - 'INSTRUMENTED_END_ASYNC_FOR': 248, - 'INSTRUMENTED_LOAD_SUPER_ATTR': 249, - 'INSTRUMENTED_CALL': 250, - 'INSTRUMENTED_CALL_KW': 251, - 'INSTRUMENTED_CALL_FUNCTION_EX': 252, - 'INSTRUMENTED_JUMP_BACKWARD': 253, - 'ANNOTATIONS_PLACEHOLDER': 256, - 'JUMP': 257, - 'JUMP_IF_FALSE': 258, - 'JUMP_IF_TRUE': 259, - 'JUMP_NO_INTERRUPT': 260, - 'LOAD_CLOSURE': 261, - 'POP_BLOCK': 262, - 'SETUP_CLEANUP': 263, - 'SETUP_FINALLY': 264, - 'SETUP_WITH': 265, - 'STORE_FAST_MAYBE_NULL': 266, -} +opmap = frozendict( + CACHE=0, + RESERVED=17, + RESUME=128, + INSTRUMENTED_LINE=253, + ENTER_EXECUTOR=254, + TRACE_RECORD=255, + BINARY_SLICE=1, + BUILD_TEMPLATE=2, + CALL_FUNCTION_EX=4, + CHECK_EG_MATCH=5, + CHECK_EXC_MATCH=6, + CLEANUP_THROW=7, + DELETE_SUBSCR=8, + END_FOR=9, + END_SEND=10, + EXIT_INIT_CHECK=11, + FORMAT_SIMPLE=12, + FORMAT_WITH_SPEC=13, + GET_AITER=14, + GET_ANEXT=15, + GET_LEN=16, + INTERPRETER_EXIT=18, + LOAD_BUILD_CLASS=19, + LOAD_LOCALS=20, + MAKE_FUNCTION=21, + MATCH_KEYS=22, + MATCH_MAPPING=23, + MATCH_SEQUENCE=24, + NOP=25, + NOT_TAKEN=26, + POP_EXCEPT=27, + POP_ITER=28, + POP_TOP=29, + PUSH_EXC_INFO=30, + PUSH_NULL=31, + RETURN_GENERATOR=32, + RETURN_VALUE=33, + SETUP_ANNOTATIONS=34, + STORE_SLICE=35, + STORE_SUBSCR=36, + TO_BOOL=37, + UNARY_INVERT=38, + UNARY_NEGATIVE=39, + UNARY_NOT=40, + WITH_EXCEPT_START=41, + BINARY_OP=42, + BUILD_INTERPOLATION=43, + BUILD_LIST=44, + BUILD_MAP=45, + BUILD_SET=46, + BUILD_SLICE=47, + BUILD_STRING=48, + BUILD_TUPLE=49, + CALL=50, + CALL_INTRINSIC_1=51, + CALL_INTRINSIC_2=52, + CALL_KW=53, + COMPARE_OP=54, + CONTAINS_OP=55, + CONVERT_VALUE=56, + COPY=57, + COPY_FREE_VARS=58, + DELETE_ATTR=59, + DELETE_DEREF=60, + DELETE_FAST=61, + DELETE_GLOBAL=62, + DELETE_NAME=63, + DICT_MERGE=64, + DICT_UPDATE=65, + END_ASYNC_FOR=66, + EXTENDED_ARG=67, + FOR_ITER=68, + GET_AWAITABLE=69, + GET_ITER=70, + IMPORT_FROM=71, + IMPORT_NAME=72, + IS_OP=73, + JUMP_BACKWARD=74, + JUMP_BACKWARD_NO_INTERRUPT=75, + JUMP_FORWARD=76, + LIST_APPEND=77, + LIST_EXTEND=78, + LOAD_ATTR=79, + LOAD_COMMON_CONSTANT=80, + LOAD_CONST=81, + LOAD_DEREF=82, + LOAD_FAST=83, + LOAD_FAST_AND_CLEAR=84, + LOAD_FAST_BORROW=85, + LOAD_FAST_BORROW_LOAD_FAST_BORROW=86, + LOAD_FAST_CHECK=87, + LOAD_FAST_LOAD_FAST=88, + LOAD_FROM_DICT_OR_DEREF=89, + LOAD_FROM_DICT_OR_GLOBALS=90, + LOAD_GLOBAL=91, + LOAD_NAME=92, + LOAD_SMALL_INT=93, + LOAD_SPECIAL=94, + LOAD_SUPER_ATTR=95, + MAKE_CELL=96, + MAP_ADD=97, + MATCH_CLASS=98, + POP_JUMP_IF_FALSE=99, + POP_JUMP_IF_NONE=100, + POP_JUMP_IF_NOT_NONE=101, + POP_JUMP_IF_TRUE=102, + RAISE_VARARGS=103, + RERAISE=104, + SEND=105, + SET_ADD=106, + SET_FUNCTION_ATTRIBUTE=107, + SET_UPDATE=108, + STORE_ATTR=109, + STORE_DEREF=110, + STORE_FAST=111, + STORE_FAST_LOAD_FAST=112, + STORE_FAST_STORE_FAST=113, + STORE_GLOBAL=114, + STORE_NAME=115, + SWAP=116, + UNPACK_EX=117, + UNPACK_SEQUENCE=118, + YIELD_VALUE=119, + INSTRUMENTED_END_FOR=233, + INSTRUMENTED_POP_ITER=234, + INSTRUMENTED_END_SEND=235, + INSTRUMENTED_FOR_ITER=236, + INSTRUMENTED_INSTRUCTION=237, + INSTRUMENTED_JUMP_FORWARD=238, + INSTRUMENTED_NOT_TAKEN=239, + INSTRUMENTED_POP_JUMP_IF_TRUE=240, + INSTRUMENTED_POP_JUMP_IF_FALSE=241, + INSTRUMENTED_POP_JUMP_IF_NONE=242, + INSTRUMENTED_POP_JUMP_IF_NOT_NONE=243, + INSTRUMENTED_RESUME=244, + INSTRUMENTED_RETURN_VALUE=245, + INSTRUMENTED_YIELD_VALUE=246, + INSTRUMENTED_END_ASYNC_FOR=247, + INSTRUMENTED_LOAD_SUPER_ATTR=248, + INSTRUMENTED_CALL=249, + INSTRUMENTED_CALL_KW=250, + INSTRUMENTED_CALL_FUNCTION_EX=251, + INSTRUMENTED_JUMP_BACKWARD=252, + ANNOTATIONS_PLACEHOLDER=256, + JUMP=257, + JUMP_IF_FALSE=258, + JUMP_IF_TRUE=259, + JUMP_NO_INTERRUPT=260, + LOAD_CLOSURE=261, + POP_BLOCK=262, + SETUP_CLEANUP=263, + SETUP_FINALLY=264, + SETUP_WITH=265, + STORE_FAST_MAYBE_NULL=266, +) -HAVE_ARGUMENT = 43 -MIN_INSTRUMENTED_OPCODE = 234 +HAVE_ARGUMENT = 41 +MIN_INSTRUMENTED_OPCODE = 233 diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index 0cb064fcd79..29b89e311cb 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -17,7 +17,8 @@ _UNIVERSAL_CONFIG_VARS = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', 'BASECFLAGS', 'BLDSHARED', 'LDSHARED', 'CC', 'CXX', 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS', - 'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS') + 'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS', + 'PY_CORE_EXE_LDFLAGS') # configuration variables that may contain compiler calls _COMPILER_CONFIG_VARS = ('BLDSHARED', 'LDSHARED', 'CC', 'CXX') diff --git a/Lib/_py_warnings.py b/Lib/_py_warnings.py index 2e7113a637a..81a386c4487 100644 --- a/Lib/_py_warnings.py +++ b/Lib/_py_warnings.py @@ -369,9 +369,15 @@ def _setoption(arg): if message or module: import re if message: - message = re.escape(message) + if len(message) >= 2 and message[0] == message[-1] == '/': + message = message[1:-1] + else: + message = re.escape(message) if module: - module = re.escape(module) + r'\z' + if len(module) >= 2 and module[0] == module[-1] == '/': + module = module[1:-1] + else: + module = re.escape(module) + r'\z' if lineno: try: lineno = int(lineno) @@ -381,7 +387,23 @@ def _setoption(arg): raise _wm._OptionError("invalid lineno %r" % (lineno,)) from None else: lineno = 0 - _wm.filterwarnings(action, message, category, module, lineno) + try: + _wm.filterwarnings(action, message, category, module, lineno) + except re.PatternError if message or module else (): + if message: + try: + re.compile(message) + except re.PatternError: + raise _wm._OptionError(f"invalid regular expression for " + f"message: {message!r}") from None + if module: + try: + re.compile(module) + except re.PatternError: + raise _wm._OptionError(f"invalid regular expression for " + f"module: {module!r}") from None + # Should never happen. + raise # Helper for _setoption() @@ -498,14 +520,43 @@ def warn(message, category=None, stacklevel=1, source=None, ) +def _match_filename(pattern, filename, *, MS_WINDOWS=(sys.platform == 'win32')): + if not filename: + return pattern.match('') is not None + if filename[0] == '<' and filename[-1] == '>': + return pattern.match(filename) is not None + + is_py = (filename[-3:].lower() == '.py' + if MS_WINDOWS else + filename.endswith('.py')) + if is_py: + filename = filename[:-3] + if pattern.match(filename): # for backward compatibility + return True + if MS_WINDOWS: + if not is_py and filename[-4:].lower() == '.pyw': + filename = filename[:-4] + is_py = True + if is_py and filename[-9:].lower() in (r'\__init__', '/__init__'): + filename = filename[:-9] + filename = filename.replace('\\', '/') + else: + if is_py and filename.endswith('/__init__'): + filename = filename[:-9] + filename = filename.replace('/', '.') + i = 0 + while True: + if pattern.match(filename, i): + return True + i = filename.find('.', i) + 1 + if not i: + return False + + def warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, source=None): lineno = int(lineno) - if module is None: - module = filename or "" - if module[-3:].lower() == ".py": - module = module[:-3] # XXX What about leading pathname? if isinstance(message, Warning): text = str(message) category = message.__class__ @@ -527,9 +578,11 @@ def warn_explicit(message, category, filename, lineno, action, msg, cat, mod, ln = item if ((msg is None or msg.match(text)) and issubclass(category, cat) and - (mod is None or mod.match(module)) and - (ln == 0 or lineno == ln)): - break + (ln == 0 or lineno == ln) and + (mod is None or (_match_filename(mod, filename) + if module is None else + mod.match(module)))): + break else: action = _wm.defaultaction # Early exit actions @@ -592,6 +645,9 @@ def __str__(self): "line : %r}" % (self.message, self._category_name, self.filename, self.lineno, self.line)) + def __repr__(self): + return f'<{type(self).__qualname__} {self}>' + class catch_warnings(object): @@ -647,8 +703,8 @@ def __enter__(self): context = None self._filters = self._module.filters self._module.filters = self._filters[:] - self._showwarning = self._module.showwarning self._showwarnmsg_impl = self._module._showwarnmsg_impl + self._showwarning = self._module.showwarning self._module._filters_mutated_lock_held() if self._record: if _use_context: @@ -656,9 +712,9 @@ def __enter__(self): else: log = [] self._module._showwarnmsg_impl = log.append - # Reset showwarning() to the default implementation to make sure - # that _showwarnmsg() calls _showwarnmsg_impl() - self._module.showwarning = self._module._showwarning_orig + # Reset showwarning() to the default implementation to make sure + # that _showwarnmsg() calls _showwarnmsg_impl() + self._module.showwarning = self._module._showwarning_orig else: log = None if self._filter is not None: @@ -673,8 +729,8 @@ def __exit__(self, *exc_info): self._module._warnings_context.set(self._saved_context) else: self._module.filters = self._filters - self._module.showwarning = self._showwarning self._module._showwarnmsg_impl = self._showwarnmsg_impl + self._module.showwarning = self._showwarning self._module._filters_mutated_lock_held() diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index ab1b1722b7c..b6d68f23728 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -1072,7 +1072,11 @@ def fromisocalendar(cls, year, week, day): @classmethod def strptime(cls, date_string, format): - """Parse string according to the given date format (like time.strptime()).""" + """Parse string according to the given date format (like time.strptime()). + + For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes + """ import _strptime return _strptime._strptime_datetime_date(cls, date_string, format) @@ -1109,6 +1113,8 @@ def strftime(self, format): Format using strftime(). Example: "%d/%m/%Y, %H:%M:%S" + For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes """ return _wrap_strftime(self, format, self.timetuple()) @@ -1456,8 +1462,13 @@ def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold return self @classmethod + def strptime(cls, date_string, format): - """Parse string according to the given time format (like time.strptime()).""" + """Parse string according to the given time format (like time.strptime()). + + For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes + """ import _strptime return _strptime._strptime_datetime_time(cls, date_string, format) @@ -1650,6 +1661,9 @@ def fromisoformat(cls, time_string): def strftime(self, format): """Format using strftime(). The date part of the timestamp passed to underlying strftime should not be used. + + For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes """ # The year must be >= 1000 else Python's strftime implementation # can raise a bogus exception. @@ -2198,7 +2212,11 @@ def __str__(self): @classmethod def strptime(cls, date_string, format): - """Parse string according to the given date and time format (like time.strptime()).""" + """Parse string according to the given time format (like time.strptime()). + + For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes + """ import _strptime return _strptime._strptime_datetime_datetime(cls, date_string, format) diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 9b8e42a2342..ef889ea0cc8 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -47,13 +47,16 @@ 'HAVE_THREADS', # C version: compile time choice that enables the coroutine local context - 'HAVE_CONTEXTVAR' + 'HAVE_CONTEXTVAR', + + # Highest version of the spec this module complies with + 'SPEC_VERSION', ] __xname__ = __name__ # sys.modules lookup (--without-threads) __name__ = 'decimal' # For pickling -__version__ = '1.70' # Highest version of the spec this complies with - # See http://speleotrove.com/decimal/ +SPEC_VERSION = '1.70' # Highest version of the spec this complies with + # See https://speleotrove.com/decimal/decarith.html __libmpdec_version__ = "2.4.2" # compatible libmpdec version import math as _math @@ -3340,7 +3343,10 @@ def _fill_logical(self, context, opa, opb): return opa, opb def logical_and(self, other, context=None): - """Applies an 'and' operation between self and other's digits.""" + """Applies an 'and' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -3357,14 +3363,20 @@ def logical_and(self, other, context=None): return _dec_from_triple(0, result.lstrip('0') or '0', 0) def logical_invert(self, context=None): - """Invert all its digits.""" + """Invert all its digits. + + The self must be logical number. + """ if context is None: context = getcontext() return self.logical_xor(_dec_from_triple(0,'1'*context.prec,0), context) def logical_or(self, other, context=None): - """Applies an 'or' operation between self and other's digits.""" + """Applies an 'or' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -3381,7 +3393,10 @@ def logical_or(self, other, context=None): return _dec_from_triple(0, result.lstrip('0') or '0', 0) def logical_xor(self, other, context=None): - """Applies an 'xor' operation between self and other's digits.""" + """Applies an 'xor' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -6387,3 +6402,11 @@ def _format_number(is_negative, intpart, fracpart, exp, spec): # _PyHASH_10INV is the inverse of 10 modulo the prime _PyHASH_MODULUS _PyHASH_10INV = pow(10, _PyHASH_MODULUS - 2, _PyHASH_MODULUS) del sys + +def __getattr__(name): + if name == "__version__": + from warnings import _deprecated + + _deprecated("__version__", remove=(3, 20)) + return SPEC_VERSION + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 5db8ce9244b..3306c8a2747 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -546,7 +546,7 @@ def nreadahead(): res += b if res.endswith(b"\n"): break - return bytes(res) + return res.take_bytes() def __iter__(self): self._checkClosed() @@ -617,8 +617,10 @@ def read(self, size=-1): n = self.readinto(b) if n is None: return None + if n < 0 or n > len(b): + raise ValueError(f"readinto returned {n} outside buffer size {len(b)}") del b[n:] - return bytes(b) + return b.take_bytes() def readall(self): """Read until EOF, using multiple read() call.""" @@ -626,7 +628,7 @@ def readall(self): while data := self.read(DEFAULT_BUFFER_SIZE): res += data if res: - return bytes(res) + return res.take_bytes() else: # b'' or None return data @@ -947,23 +949,24 @@ def read1(self, size=-1): return self.read(size) def write(self, b): - if self.closed: - raise ValueError("write to closed file") if isinstance(b, str): raise TypeError("can't write str to binary stream") with memoryview(b) as view: - n = view.nbytes # Size of any bytes-like object - if n == 0: - return 0 + if self.closed: + raise ValueError("write to closed file") - with self._lock: - pos = self._pos - if pos > len(self._buffer): - # Pad buffer to pos with null bytes. - self._buffer.resize(pos) - self._buffer[pos:pos + n] = b - self._pos += n - return n + n = view.nbytes # Size of any bytes-like object + if n == 0: + return 0 + + with self._lock: + pos = self._pos + if pos > len(self._buffer): + # Pad buffer to pos with null bytes. + self._buffer.resize(pos) + self._buffer[pos:pos + n] = view + self._pos += n + return n def seek(self, pos, whence=0): if self.closed: @@ -1498,6 +1501,7 @@ class FileIO(RawIOBase): _writable = False _appending = False _seekable = None + _truncate = False _closefd = True def __init__(self, file, mode='r', closefd=True, opener=None): @@ -1553,6 +1557,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None): flags = 0 elif 'w' in mode: self._writable = True + self._truncate = True flags = os.O_CREAT | os.O_TRUNC elif 'a' in mode: self._writable = True @@ -1734,7 +1739,7 @@ def readall(self): assert len(result) - bytes_read >= 1, \ "os.readinto buffer size 0 will result in erroneous EOF / returns 0" result.resize(bytes_read) - return bytes(result) + return result.take_bytes() def readinto(self, buffer): """Same as RawIOBase.readinto().""" @@ -1877,7 +1882,10 @@ def mode(self): return 'ab' elif self._readable: if self._writable: - return 'rb+' + if self._truncate: + return 'wb+' + else: + return 'rb+' else: return 'rb' else: diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index 1e9462a4215..a22b0297b24 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -1,9 +1,13 @@ from __future__ import annotations +import importlib +import os import pkgutil +import re import sys import token import tokenize +from importlib.machinery import FileFinder from io import StringIO from contextlib import contextmanager from dataclasses import dataclass @@ -13,7 +17,30 @@ TYPE_CHECKING = False if TYPE_CHECKING: + from types import ModuleType from typing import Any, Iterable, Iterator, Mapping + from .types import CompletionAction + + +HARDCODED_SUBMODULES = { + # Standard library submodules that are not detected by pkgutil.iter_modules + # but can be imported, so should be proposed in completion + "collections": ["abc"], + "math": ["integer"], + "os": ["path"], + "xml.parsers.expat": ["errors", "model"], +} + +AUTO_IMPORT_DENYLIST = { + # Standard library modules/submodules that have import side effects + # and must not be automatically imported to complete attributes + re.compile(r"antigravity"), # Calls webbrowser.open + re.compile(r"idlelib\..+"), # May open IDLE GUI + re.compile(r"test\..+"), # Various side-effects + re.compile(r"this"), # Prints to stdout + re.compile(r"_ios_support"), # Spawns a subprocess + re.compile(r".+\.__main__"), # Should not be imported +} def make_default_module_completer() -> ModuleCompleter: @@ -40,10 +67,17 @@ class ModuleCompleter: def __init__(self, namespace: Mapping[str, Any] | None = None) -> None: self.namespace = namespace or {} self._global_cache: list[pkgutil.ModuleInfo] = [] + self._failed_imports: set[str] = set() self._curr_sys_path: list[str] = sys.path[:] + self._stdlib_path = os.path.dirname(importlib.__path__[0]) - def get_completions(self, line: str) -> list[str] | None: - """Return the next possible import completions for 'line'.""" + def get_completions(self, line: str) -> tuple[list[str], CompletionAction | None] | None: + """Return the next possible import completions for 'line'. + + For attributes completion, if the module to complete from is not + imported, also return an action (prompt + callback to run if the + user press TAB again) to import the module. + """ result = ImportParser(line).parse() if not result: return None @@ -52,24 +86,26 @@ def get_completions(self, line: str) -> list[str] | None: except Exception: # Some unexpected error occurred, make it look like # no completions are available - return [] + return [], None - def complete(self, from_name: str | None, name: str | None) -> list[str]: + def complete(self, from_name: str | None, name: str | None) -> tuple[list[str], CompletionAction | None]: if from_name is None: # import x.y.z assert name is not None path, prefix = self.get_path_and_prefix(name) modules = self.find_modules(path, prefix) - return [self.format_completion(path, module) for module in modules] + return [self.format_completion(path, module) for module in modules], None if name is None: # from x.y.z path, prefix = self.get_path_and_prefix(from_name) modules = self.find_modules(path, prefix) - return [self.format_completion(path, module) for module in modules] + return [self.format_completion(path, module) for module in modules], None # from x.y import z - return self.find_modules(from_name, name) + submodules = self.find_modules(from_name, name) + attributes, action = self.find_attributes(from_name, name) + return sorted({*submodules, *attributes}), action def find_modules(self, path: str, prefix: str) -> list[str]: """Find all modules under 'path' that start with 'prefix'.""" @@ -87,20 +123,74 @@ def _find_modules(self, path: str, prefix: str) -> list[str]: if self.is_suggestion_match(module.name, prefix)] return sorted(builtin_modules + third_party_modules) - if path.startswith('.'): - # Convert relative path to absolute path - package = self.namespace.get('__package__', '') - path = self.resolve_relative_name(path, package) # type: ignore[assignment] - if path is None: - return [] + path = self._resolve_relative_path(path) # type: ignore[assignment] + if path is None: + return [] modules: Iterable[pkgutil.ModuleInfo] = self.global_cache + imported_module = sys.modules.get(path.split('.')[0]) + if imported_module: + # Filter modules to those whose name and specs match the + # imported module to avoid invalid suggestions + spec = imported_module.__spec__ + if spec: + def _safe_find_spec(mod: pkgutil.ModuleInfo) -> bool: + try: + return mod.module_finder.find_spec(mod.name, None) == spec + except Exception: + return False + modules = [mod for mod in modules + if mod.name == spec.name + and _safe_find_spec(mod)] + else: + modules = [] + + is_stdlib_import: bool | None = None for segment in path.split('.'): modules = [mod_info for mod_info in modules if mod_info.ispkg and mod_info.name == segment] + if is_stdlib_import is None: + # Top-level import decide if we import from stdlib or not + is_stdlib_import = all( + self._is_stdlib_module(mod_info) for mod_info in modules + ) modules = self.iter_submodules(modules) - return [module.name for module in modules - if self.is_suggestion_match(module.name, prefix)] + + module_names = [module.name for module in modules] + if is_stdlib_import: + module_names.extend(HARDCODED_SUBMODULES.get(path, ())) + return [module_name for module_name in module_names + if self.is_suggestion_match(module_name, prefix)] + + def _is_stdlib_module(self, module_info: pkgutil.ModuleInfo) -> bool: + return (isinstance(module_info.module_finder, FileFinder) + and module_info.module_finder.path == self._stdlib_path) + + def find_attributes(self, path: str, prefix: str) -> tuple[list[str], CompletionAction | None]: + """Find all attributes of module 'path' that start with 'prefix'.""" + attributes, action = self._find_attributes(path, prefix) + # Filter out invalid attribute names + # (for example those containing dashes that cannot be imported with 'import') + return [attr for attr in attributes if attr.isidentifier()], action + + def _find_attributes(self, path: str, prefix: str) -> tuple[list[str], CompletionAction | None]: + path = self._resolve_relative_path(path) # type: ignore[assignment] + if path is None: + return [], None + + imported_module = sys.modules.get(path) + if not imported_module: + if path in self._failed_imports: # Do not propose to import again + return [], None + imported_module = self._maybe_import_module(path) + if not imported_module: + return [], self._get_import_completion_action(path) + try: + module_attributes = dir(imported_module) + except Exception: + module_attributes = [] + return [attr_name for attr_name in module_attributes + if self.is_suggestion_match(attr_name, prefix)], None def is_suggestion_match(self, module_name: str, prefix: str) -> bool: if prefix: @@ -146,6 +236,13 @@ def format_completion(self, path: str, module: str) -> str: return f'{path}{module}' return f'{path}.{module}' + def _resolve_relative_path(self, path: str) -> str | None: + """Resolve a relative import path to absolute. Returns None if unresolvable.""" + if path.startswith('.'): + package = self.namespace.get('__package__', '') + return self.resolve_relative_name(path, package) + return path + def resolve_relative_name(self, name: str, package: str) -> str | None: """Resolve a relative module name to an absolute name. @@ -169,10 +266,40 @@ def global_cache(self) -> list[pkgutil.ModuleInfo]: """Global module cache""" if not self._global_cache or self._curr_sys_path != sys.path: self._curr_sys_path = sys.path[:] - # print('getting packages') self._global_cache = list(pkgutil.iter_modules()) + self._failed_imports.clear() # retry on sys.path change return self._global_cache + def _maybe_import_module(self, fqname: str) -> ModuleType | None: + if any(pattern.fullmatch(fqname) for pattern in AUTO_IMPORT_DENYLIST): + # Special-cased modules with known import side-effects + return None + root = fqname.split(".")[0] + mod_info = next((m for m in self.global_cache if m.name == root), None) + if not mod_info or not self._is_stdlib_module(mod_info): + # Only import stdlib modules (no risk of import side-effects) + return None + try: + return importlib.import_module(fqname) + except Exception: + sys.modules.pop(fqname, None) # Clean half-imported module + return None + + def _get_import_completion_action(self, path: str) -> CompletionAction: + prompt = ("[ module not imported, press again to import it " + "and propose attributes ]") + + def _do_import() -> str | None: + try: + importlib.import_module(path) + return None + except Exception as exc: + sys.modules.pop(path, None) # Clean half-imported module + self._failed_imports.add(path) + return f"[ error during import: {exc} ]" + + return (prompt, _do_import) + class ImportParser: """ diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 50c824995d8..e79fbfa6bb0 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -22,6 +22,7 @@ from __future__ import annotations import os import time +from typing import TYPE_CHECKING # Categories of actions: # killing @@ -32,10 +33,11 @@ # finishing # [completion] +from .render import RenderedScreen from .trace import trace # types -if False: +if TYPE_CHECKING: from .historical_reader import HistoricalReader @@ -74,7 +76,7 @@ def kill_range(self, start: int, end: int) -> None: else: r.kill_ring.append(text) r.pos = start - r.dirty = True + r.invalidate_buffer(start) class YankCommand(Command): @@ -125,24 +127,27 @@ def do(self) -> None: r.arg = 10 * r.arg - d else: r.arg = 10 * r.arg + d - r.dirty = True + r.invalidate_prompt() class clear_screen(Command): def do(self) -> None: r = self.reader + trace("command.clear_screen") r.console.clear() - r.dirty = True + r.invalidate_full() class refresh(Command): def do(self) -> None: - self.reader.dirty = True + trace("command.refresh") + self.reader.invalidate_full() class repaint(Command): def do(self) -> None: - self.reader.dirty = True + trace("command.repaint") + self.reader.invalidate_full() self.reader.console.repaint() @@ -208,9 +213,10 @@ def do(self) -> None: repl = len(r.kill_ring[-1]) r.kill_ring.insert(0, r.kill_ring.pop()) t = r.kill_ring[-1] + start = r.pos - repl b[r.pos - repl : r.pos] = t r.pos = r.pos - repl + len(t) - r.dirty = True + r.invalidate_buffer(start) class interrupt(FinishCommand): @@ -242,8 +248,9 @@ def do(self) -> None: r.console.prepare() r.pos = p # r.posxy = 0, 0 # XXX this is invalid - r.dirty = True - r.console.screen = [] + r.invalidate_full() + trace("command.suspend sync_rendered_screen") + r.console.sync_rendered_screen(RenderedScreen.empty(), r.console.posxy) class up(MotionCommand): @@ -369,6 +376,7 @@ class self_insert(EditCommand): def do(self) -> None: r = self.reader text = self.event * r.get_arg() + start = r.pos r.insert(text) if r.paste_mode: data = "" @@ -376,7 +384,7 @@ def do(self) -> None: data += ev.data if data: r.insert(data) - r.last_refresh_cache.invalidated = True + r.invalidate_buffer(start) class insert_nl(EditCommand): @@ -400,40 +408,49 @@ def do(self) -> None: del b[s] b.insert(t, c) r.pos = t - r.dirty = True + r.invalidate_buffer(s) class backspace(EditCommand): def do(self) -> None: r = self.reader b = r.buffer + changed_from: int | None = None for i in range(r.get_arg()): if r.pos > 0: r.pos -= 1 del b[r.pos] - r.dirty = True + changed_from = r.pos if changed_from is None else min(changed_from, r.pos) else: self.reader.error("can't backspace at start") + if changed_from is not None: + r.invalidate_buffer(changed_from) class delete(EditCommand): def do(self) -> None: r = self.reader b = r.buffer - if ( - r.pos == 0 - and len(b) == 0 # this is something of a hack - and self.event[-1] == "\004" - ): - r.update_screen() - r.console.finish() - raise EOFError + if self.event[-1] == "\004": + if b and b[-1].endswith("\n"): + self.finish = True + elif ( + r.pos == 0 + and len(b) == 0 # this is something of a hack + ): + r.update_screen() + r.console.finish() + raise EOFError + + changed_from: int | None = None for i in range(r.get_arg()): if r.pos != len(b): del b[r.pos] - r.dirty = True + changed_from = r.pos if changed_from is None else min(changed_from, r.pos) else: self.reader.error("end of buffer") + if changed_from is not None: + r.invalidate_buffer(changed_from) class accept(FinishCommand): @@ -447,6 +464,7 @@ def do(self) -> None: with self.reader.suspend(): self.reader.msg = _sitebuiltins._Helper()() # type: ignore[assignment] + self.reader.invalidate_prompt() class invalid_key(Command): @@ -467,22 +485,24 @@ def do(self) -> None: from .pager import get_pager from site import gethistoryfile + # After the pager exits, the screen state is unknown (Unix may + # restore via alternate screen, Windows shows pager output). + # Clear and force a full redraw at the end for consistency. + self.reader.console.clear() + history = os.linesep.join(self.reader.history[:]) self.reader.console.restore() pager = get_pager() pager(history, gethistoryfile()) self.reader.console.prepare() - # We need to copy over the state so that it's consistent between - # console and reader, and console does not overwrite/append stuff - self.reader.console.screen = self.reader.screen.copy() - self.reader.console.posxy = self.reader.cxy + self.reader.invalidate_full() class paste_mode(Command): def do(self) -> None: self.reader.paste_mode = not self.reader.paste_mode - self.reader.dirty = True + self.reader.invalidate_prompt() class perform_bracketed_paste(Command): @@ -499,4 +519,3 @@ def do(self) -> None: s=time.time() - start, ) self.reader.insert(data.replace(done, "")) - self.reader.last_refresh_cache.invalidated = True diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py index 9d2d43be514..f783e8db36b 100644 --- a/Lib/_pyrepl/completing_reader.py +++ b/Lib/_pyrepl/completing_reader.py @@ -21,16 +21,18 @@ from __future__ import annotations from dataclasses import dataclass, field +from typing import TYPE_CHECKING import re from . import commands, console, reader +from .render import RenderLine, ScreenOverlay from .reader import Reader # types Command = commands.Command -if False: - from .types import KeySpec, CommandName +if TYPE_CHECKING: + from .types import CommandName, CompletionAction, Keymap, KeySpec def prefix(wordlist: list[str], j: int = 0) -> str: @@ -168,39 +170,68 @@ def do(self) -> None: r: CompletingReader r = self.reader # type: ignore[assignment] last_is_completer = r.last_command_is(self.__class__) + if r.cmpltn_action: + if last_is_completer: # double-tab: execute action + msg = r.cmpltn_action[1]() + r.cmpltn_action = None # consumed + if msg: + r.msg = msg + r.cmpltn_message_visible = True + r.invalidate_message() + else: # other input since last tab: cancel action + r.cmpltn_action = None + immutable_completions = r.assume_immutable_completions completions_unchangable = last_is_completer and immutable_completions stem = r.get_stem() if not completions_unchangable: - r.cmpltn_menu_choices = r.get_completions(stem) + r.cmpltn_menu_choices, r.cmpltn_action = r.get_completions(stem) completions = r.cmpltn_menu_choices if not completions: - r.error("no matches") + if not r.cmpltn_action: + r.error("no matches") elif len(completions) == 1: - if completions_unchangable and len(completions[0]) == len(stem): + completion = stripcolor(completions[0]) + if completions_unchangable and len(completion) == len(stem): r.msg = "[ sole completion ]" - r.dirty = True - r.insert(completions[0][len(stem):]) + r.cmpltn_message_visible = True + r.invalidate_message() + r.insert(completion[len(stem):]) else: - p = prefix(completions, len(stem)) + clean_completions = [stripcolor(word) for word in completions] + p = prefix(clean_completions, len(stem)) if p: r.insert(p) if last_is_completer: r.cmpltn_menu_visible = True - r.cmpltn_message_visible = False r.cmpltn_menu, r.cmpltn_menu_end = build_menu( r.console, completions, r.cmpltn_menu_end, r.use_brackets, r.sort_in_column) - r.dirty = True + if r.msg: + r.msg = "" + r.cmpltn_message_visible = False + r.invalidate_message() + r.invalidate_overlay() elif not r.cmpltn_menu_visible: - r.cmpltn_message_visible = True - if stem + p in completions: + if stem + p in clean_completions: r.msg = "[ complete but not unique ]" - r.dirty = True + r.cmpltn_message_visible = True + r.invalidate_message() else: r.msg = "[ not unique ]" - r.dirty = True + r.cmpltn_message_visible = True + r.invalidate_message() + + if r.cmpltn_action: + if r.msg and r.cmpltn_message_visible: + # There is already a message (eg. [ not unique ]) that + # would conflict for next tab: cancel action + r.cmpltn_action = None + else: + r.msg = r.cmpltn_action[0] + r.cmpltn_message_visible = True + r.invalidate_message() class self_insert(commands.self_insert): @@ -215,11 +246,12 @@ def do(self) -> None: r.cmpltn_reset() else: completions = [w for w in r.cmpltn_menu_choices - if w.startswith(stem)] + if stripcolor(w).startswith(stem)] if completions: r.cmpltn_menu, r.cmpltn_menu_end = build_menu( r.console, completions, 0, r.use_brackets, r.sort_in_column) + r.invalidate_overlay() else: r.cmpltn_reset() @@ -240,6 +272,7 @@ class CompletingReader(Reader): cmpltn_message_visible: bool = field(init=False) cmpltn_menu_end: int = field(init=False) cmpltn_menu_choices: list[str] = field(init=False) + cmpltn_action: CompletionAction | None = field(init=False) def __post_init__(self) -> None: super().__post_init__() @@ -248,7 +281,7 @@ def __post_init__(self) -> None: self.commands[c.__name__] = c self.commands[c.__name__.replace('_', '-')] = c - def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + def collect_keymap(self) -> Keymap: return super().collect_keymap() + ( (r'\t', 'complete'),) @@ -257,30 +290,30 @@ def after_command(self, cmd: Command) -> None: if not isinstance(cmd, (complete, self_insert)): self.cmpltn_reset() - def calc_screen(self) -> list[str]: - screen = super().calc_screen() - if self.cmpltn_menu_visible: - # We display the completions menu below the current prompt - ly = self.lxy[1] + 1 - screen[ly:ly] = self.cmpltn_menu - # If we're not in the middle of multiline edit, don't append to screeninfo - # since that screws up the position calculation in pos2xy function. - # This is a hack to prevent the cursor jumping - # into the completions menu when pressing left or down arrow. - if self.pos != len(self.buffer): - self.screeninfo[ly:ly] = [(0, [])]*len(self.cmpltn_menu) - return screen + def get_screen_overlays(self) -> tuple[ScreenOverlay, ...]: + if not self.cmpltn_menu_visible: + return () + return ( + ScreenOverlay( + self.lxy[1] + 1, + tuple(RenderLine.from_rendered_text(line) for line in self.cmpltn_menu), + insert=True, + ), + ) def finish(self) -> None: super().finish() self.cmpltn_reset() def cmpltn_reset(self) -> None: + if getattr(self, "cmpltn_menu_visible", False): + self.invalidate_overlay() self.cmpltn_menu = [] self.cmpltn_menu_visible = False self.cmpltn_message_visible = False self.cmpltn_menu_end = 0 self.cmpltn_menu_choices = [] + self.cmpltn_action = None def get_stem(self) -> str: st = self.syntax_table @@ -291,8 +324,8 @@ def get_stem(self) -> str: p -= 1 return ''.join(b[p+1:self.pos]) - def get_completions(self, stem: str) -> list[str]: - return [] + def get_completions(self, stem: str) -> tuple[list[str], CompletionAction | None]: + return [], None def get_line(self) -> str: """Return the current line until the cursor position.""" diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index e0535d50396..2a53d5ff581 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -19,23 +19,25 @@ from __future__ import annotations +import os import _colorize from abc import ABC, abstractmethod import ast import code import linecache -from dataclasses import dataclass, field -import os.path +from dataclasses import dataclass import re import sys +from typing import TYPE_CHECKING - -TYPE_CHECKING = False +from .render import RenderedScreen +from .trace import trace if TYPE_CHECKING: - from typing import IO - from typing import Callable + from typing import Callable, IO + + from .types import CursorXY @dataclass @@ -47,10 +49,17 @@ class Event: @dataclass class Console(ABC): - posxy: tuple[int, int] - screen: list[str] = field(default_factory=list) + posxy: CursorXY = (0, 0) height: int = 25 width: int = 80 + _redraw_debug_palette: tuple[str, ...] = ( + "\x1b[41m", + "\x1b[42m", + "\x1b[43m", + "\x1b[44m", + "\x1b[45m", + "\x1b[46m", + ) def __init__( self, @@ -71,8 +80,52 @@ def __init__( else: self.output_fd = f_out.fileno() + self.posxy = (0, 0) + self.height = 25 + self.width = 80 + self._rendered_screen = RenderedScreen.empty() + self._redraw_visual_cycle = 0 + + @property + def screen(self) -> list[str]: + return list(self._rendered_screen.screen_lines) + + def sync_rendered_screen( + self, + rendered_screen: RenderedScreen, + posxy: CursorXY | None = None, + ) -> None: + if posxy is None: + posxy = rendered_screen.cursor + self.posxy = posxy + self._rendered_screen = rendered_screen + trace( + "console.sync_rendered_screen lines={lines} cursor={cursor}", + lines=len(rendered_screen.composed_lines), + cursor=posxy, + ) + + def invalidate_render_state(self) -> None: + self._rendered_screen = RenderedScreen.empty() + trace("console.invalidate_render_state") + + def begin_redraw_visualization(self) -> str | None: + if "PYREPL_VISUALIZE_REDRAWS" not in os.environ: + return None + + palette = self._redraw_debug_palette + cycle = self._redraw_visual_cycle + style = palette[cycle % len(palette)] + self._redraw_visual_cycle = cycle + 1 + trace( + "console.begin_redraw_visualization cycle={cycle} style={style!r}", + cycle=cycle, + style=style, + ) + return style + @abstractmethod - def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: ... + def refresh(self, rendered_screen: RenderedScreen) -> None: ... @abstractmethod def prepare(self) -> None: ... diff --git a/Lib/_pyrepl/content.py b/Lib/_pyrepl/content.py new file mode 100644 index 00000000000..3cb22fee84b --- /dev/null +++ b/Lib/_pyrepl/content.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from .utils import ColorSpan, StyleRef, THEME, iter_display_chars, unbracket, wlen + + +@dataclass(frozen=True, slots=True) +class ContentFragment: + """A single display character with its visual width and style. + + The body of ``>>> def greet`` becomes one fragment per character:: + + d e f g r e e t + ╰──┴──╯ ╰──┴──┴──┴──╯ + keyword (unstyled) + + e.g. ``ContentFragment("d", 1, StyleRef(tag="keyword"))``. + """ + + text: str + width: int + style: StyleRef = StyleRef() + + +@dataclass(frozen=True, slots=True) +class PromptContent: + """The prompt split into leading full-width lines and an inline portion. + + For the common ``">>> "`` prompt (no newlines):: + + >>> def greet(name): + ╰─╯ + text=">>> ", width=4, leading_lines=() + + If ``sys.ps1`` contains newlines, e.g. ``"Python 3.13\\n>>> "``:: + + Python 3.13 ← leading_lines[0] + >>> def greet(name): + ╰─╯ + text=">>> ", width=4 + """ + + leading_lines: tuple[ContentFragment, ...] + text: str + width: int + + +@dataclass(frozen=True, slots=True) +class SourceLine: + """One logical line from the editor buffer, before styling. + + Given this two-line input in the REPL:: + + >>> def greet(name): + ... return name + ▲ cursor + + The buffer ``"def greet(name):\\n return name"`` yields:: + + SourceLine(lineno=0, text="def greet(name):", + start_offset=0, has_newline=True) + SourceLine(lineno=1, text=" return name", + start_offset=17, cursor_index=14) + """ + + lineno: int + text: str + start_offset: int + has_newline: bool + cursor_index: int | None = None + + @property + def cursor_on_line(self) -> bool: + return self.cursor_index is not None + + +@dataclass(frozen=True, slots=True) +class ContentLine: + """A logical line paired with its prompt and styled body. + + For ``>>> def greet(name):``:: + + >>> def greet(name): + ╰─╯ ╰──────────────╯ + prompt body: one ContentFragment per character + """ + + source: SourceLine + prompt: PromptContent + body: tuple[ContentFragment, ...] + + +def process_prompt(prompt: str) -> PromptContent: + r"""Return prompt content with width measured without zero-width markup.""" + + prompt_text = unbracket(prompt, including_content=False) + visible_prompt = unbracket(prompt, including_content=True) + leading_lines: list[ContentFragment] = [] + + while "\n" in prompt_text: + leading_text, _, prompt_text = prompt_text.partition("\n") + visible_leading, _, visible_prompt = visible_prompt.partition("\n") + leading_lines.append(ContentFragment(leading_text, wlen(visible_leading))) + + return PromptContent(tuple(leading_lines), prompt_text, wlen(visible_prompt)) + + +def build_body_fragments( + buffer: str, + colors: list[ColorSpan] | None, + start_index: int, +) -> tuple[ContentFragment, ...]: + """Convert a line's text into styled content fragments.""" + # Two separate loops to avoid the THEME() call in the common uncolored path. + if colors is None: + return tuple( + ContentFragment( + styled_char.text, + styled_char.width, + StyleRef(), + ) + for styled_char in iter_display_chars(buffer, colors, start_index) + ) + + theme = THEME() + return tuple( + ContentFragment( + styled_char.text, + styled_char.width, + StyleRef.from_tag(styled_char.tag, theme[styled_char.tag]) + if styled_char.tag + else StyleRef(), + ) + for styled_char in iter_display_chars(buffer, colors, start_index) + ) diff --git a/Lib/_pyrepl/fancy_termios.py b/Lib/_pyrepl/fancy_termios.py index 0468b9a2670..8d5bd183f21 100644 --- a/Lib/_pyrepl/fancy_termios.py +++ b/Lib/_pyrepl/fancy_termios.py @@ -20,19 +20,25 @@ import termios -class TermState: - def __init__(self, tuples): - ( - self.iflag, - self.oflag, - self.cflag, - self.lflag, - self.ispeed, - self.ospeed, - self.cc, - ) = tuples +TYPE_CHECKING = False - def as_list(self): +if TYPE_CHECKING: + from typing import cast +else: + cast = lambda typ, val: val + + +class TermState: + def __init__(self, attrs: list[int | list[bytes]]) -> None: + self.iflag = cast(int, attrs[0]) + self.oflag = cast(int, attrs[1]) + self.cflag = cast(int, attrs[2]) + self.lflag = cast(int, attrs[3]) + self.ispeed = cast(int, attrs[4]) + self.ospeed = cast(int, attrs[5]) + self.cc = cast(list[bytes], attrs[6]) + + def as_list(self) -> list[int | list[bytes]]: return [ self.iflag, self.oflag, @@ -45,32 +51,32 @@ def as_list(self): self.cc[:], ] - def copy(self): + def copy(self) -> "TermState": return self.__class__(self.as_list()) -def tcgetattr(fd): +def tcgetattr(fd: int) -> TermState: return TermState(termios.tcgetattr(fd)) -def tcsetattr(fd, when, attrs): +def tcsetattr(fd: int, when: int, attrs: TermState) -> None: termios.tcsetattr(fd, when, attrs.as_list()) class Term(TermState): TS__init__ = TermState.__init__ - def __init__(self, fd=0): + def __init__(self, fd: int = 0) -> None: self.TS__init__(termios.tcgetattr(fd)) self.fd = fd - self.stack = [] + self.stack: list[list[int | list[bytes]]] = [] - def save(self): + def save(self) -> None: self.stack.append(self.as_list()) - def set(self, when=termios.TCSANOW): + def set(self, when: int = termios.TCSANOW) -> None: termios.tcsetattr(self.fd, when, self.as_list()) - def restore(self): + def restore(self) -> None: self.TS__init__(self.stack.pop()) self.set() diff --git a/Lib/_pyrepl/fancycompleter.py b/Lib/_pyrepl/fancycompleter.py new file mode 100644 index 00000000000..7a639afd74e --- /dev/null +++ b/Lib/_pyrepl/fancycompleter.py @@ -0,0 +1,201 @@ +# Copyright 2010-2025 Antonio Cuni +# Daniel Hahler +# +# All Rights Reserved +"""Colorful tab completion for Python prompt""" +from _colorize import ANSIColors, get_colors, get_theme +import rlcompleter +import keyword +import types + +class Completer(rlcompleter.Completer): + """ + When doing something like a.b., keep the full a.b.attr completion + stem so readline-style completion can keep refining the menu as you type. + + Optionally, display the various completions in different colors + depending on the type. + """ + def __init__( + self, + namespace=None, + *, + use_colors='auto', + consider_getitems=True, + ): + from _pyrepl import readline + rlcompleter.Completer.__init__(self, namespace) + if use_colors == 'auto': + # use colors only if we can + use_colors = get_colors().RED != "" + self.use_colors = use_colors + self.consider_getitems = consider_getitems + + if self.use_colors: + # In GNU readline, this prevents escaping of ANSI control + # characters in completion results. pyrepl's parse_and_bind() + # is a no-op, but pyrepl handles ANSI sequences natively + # via real_len()/stripcolor(). + readline.parse_and_bind('set dont-escape-ctrl-chars on') + self.theme = get_theme() + else: + self.theme = None + + if self.consider_getitems: + delims = readline.get_completer_delims() + delims = delims.replace('[', '') + delims = delims.replace(']', '') + readline.set_completer_delims(delims) + + def complete(self, text, state): + # if you press at the beginning of a line, insert an actual + # \t. Else, trigger completion. + if text == "": + return ('\t', None)[state] + else: + return rlcompleter.Completer.complete(self, text, state) + + def _callable_postfix(self, val, word): + # disable automatic insertion of '(' for global callables + return word + + def _callable_attr_postfix(self, val, word): + return rlcompleter.Completer._callable_postfix(self, val, word) + + def global_matches(self, text): + names = rlcompleter.Completer.global_matches(self, text) + prefix = commonprefix(names) + if prefix and prefix != text: + return [prefix] + + names.sort() + values = [] + for name in names: + clean_name = name.rstrip(': ') + if keyword.iskeyword(clean_name) or keyword.issoftkeyword(clean_name): + values.append(None) + else: + try: + values.append(eval(name, self.namespace)) + except Exception: + values.append(None) + if self.use_colors and names: + return self.colorize_matches(names, values) + return names + + def attr_matches(self, text): + try: + expr, attr, names, values = self._attr_matches(text) + except ValueError: + return [] + + if not names: + return [] + + if len(names) == 1: + # No coloring: when returning a single completion, readline + # inserts it directly into the prompt, so ANSI codes would + # appear as literal characters. + return [self._callable_attr_postfix(values[0], f'{expr}.{names[0]}')] + + prefix = commonprefix(names) + if prefix and prefix != attr: + return [f'{expr}.{prefix}'] # autocomplete prefix + + names = [f'{expr}.{name}' for name in names] + if self.use_colors: + return self.colorize_matches(names, values) + return names + + def _attr_matches(self, text): + expr, attr = text.rsplit('.', 1) + if '(' in expr or ')' in expr: # don't call functions + return expr, attr, [], [] + try: + thisobject = eval(expr, self.namespace) + except Exception: + return expr, attr, [], [] + + # get the content of the object, except __builtins__ + words = set(dir(thisobject)) - {'__builtins__'} + + if hasattr(thisobject, '__class__'): + words.add('__class__') + words.update(rlcompleter.get_class_members(thisobject.__class__)) + names = [] + values = [] + n = len(attr) + if attr == '': + noprefix = '_' + elif attr == '_': + noprefix = '__' + else: + noprefix = None + + # sort the words now to make sure to return completions in + # alphabetical order. It's easier to do it now, else we would need to + # sort 'names' later but make sure that 'values' in kept in sync, + # which is annoying. + words = sorted(words) + while True: + for word in words: + if ( + word[:n] == attr + and not (noprefix and word[:n+1] == noprefix) + ): + # Mirror rlcompleter's safeguards so completion does not + # call properties or reify lazy module attributes. + if isinstance(getattr(type(thisobject), word, None), property): + value = None + elif ( + isinstance(thisobject, types.ModuleType) + and isinstance( + thisobject.__dict__.get(word), + types.LazyImportType, + ) + ): + value = thisobject.__dict__.get(word) + else: + value = getattr(thisobject, word, None) + + names.append(word) + values.append(value) + if names or not noprefix: + break + if noprefix == '_': + noprefix = '__' + else: + noprefix = None + + return expr, attr, names, values + + def colorize_matches(self, names, values): + return [ + self._color_for_obj(name, obj) + for name, obj in zip(names, values) + ] + + def _color_for_obj(self, name, value): + t = type(value) + color = self._color_by_type(t) + return f"{color}{name}{ANSIColors.RESET}" + + def _color_by_type(self, t): + typename = t.__name__ + # this is needed e.g. to turn method-wrapper into method_wrapper, + # because if we want _colorize.FancyCompleter to be "dataclassable" + # our keys need to be valid identifiers. + typename = typename.replace('-', '_').replace('.', '_') + return getattr(self.theme.fancycompleter, typename, ANSIColors.RESET) + + +def commonprefix(names): + """Return the common prefix of all 'names'""" + if not names: + return '' + s1 = min(names) + s2 = max(names) + for i, c in enumerate(s1): + if c != s2[i]: + return s1[:i] + return s1 diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py index c4b95fa2e81..09b969d80bc 100644 --- a/Lib/_pyrepl/historical_reader.py +++ b/Lib/_pyrepl/historical_reader.py @@ -90,7 +90,7 @@ def do(self) -> None: if r.get_unicode() != r.history[r.historyi]: r.buffer = list(r.history[r.historyi]) r.pos = len(r.buffer) - r.dirty = True + r.invalidate_buffer(0) class first_history(commands.Command): @@ -130,10 +130,11 @@ def do(self) -> None: o = len(r.yank_arg_yanked) else: o = 0 + start = r.pos - o b[r.pos - o : r.pos] = list(w) r.yank_arg_yanked = w r.pos += len(w) - o - r.dirty = True + r.invalidate_buffer(start) class forward_history_isearch(commands.Command): @@ -142,7 +143,7 @@ def do(self) -> None: r.isearch_direction = ISEARCH_DIRECTION_FORWARDS r.isearch_start = r.historyi, r.pos r.isearch_term = "" - r.dirty = True + r.invalidate_prompt() r.push_input_trans(r.isearch_trans) @@ -150,7 +151,7 @@ class reverse_history_isearch(commands.Command): def do(self) -> None: r = self.reader r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS - r.dirty = True + r.invalidate_prompt() r.isearch_term = "" r.push_input_trans(r.isearch_trans) r.isearch_start = r.historyi, r.pos @@ -163,7 +164,7 @@ def do(self) -> None: r.pop_input_trans() r.select_item(r.isearch_start[0]) r.pos = r.isearch_start[1] - r.dirty = True + r.invalidate_prompt() class isearch_add_character(commands.Command): @@ -171,7 +172,7 @@ def do(self) -> None: r = self.reader b = r.buffer r.isearch_term += self.event[-1] - r.dirty = True + r.invalidate_prompt() p = r.pos + len(r.isearch_term) - 1 if b[p : p + 1] != [r.isearch_term[-1]]: r.isearch_next() @@ -182,7 +183,7 @@ def do(self) -> None: r = self.reader if len(r.isearch_term) > 0: r.isearch_term = r.isearch_term[:-1] - r.dirty = True + r.invalidate_prompt() else: r.error("nothing to rubout") @@ -207,7 +208,7 @@ def do(self) -> None: r.isearch_direction = ISEARCH_DIRECTION_NONE r.console.forgetinput() r.pop_input_trans() - r.dirty = True + r.invalidate_prompt() @dataclass @@ -241,7 +242,6 @@ def __post_init__(self) -> None: isearch_end, isearch_add_character, isearch_cancel, - isearch_add_character, isearch_backspace, isearch_forwards, isearch_backwards, @@ -279,8 +279,7 @@ def select_item(self, i: int) -> None: self.buffer = list(buf) self.historyi = i self.pos = len(self.buffer) - self.dirty = True - self.last_refresh_cache.invalidated = True + self.invalidate_buffer(0) def get_item(self, i: int) -> str: if i != len(self.history): @@ -358,7 +357,7 @@ def search_next(self, *, forwards: bool) -> None: if forwards and not match_prefix: self.pos = 0 self.buffer = [] - self.dirty = True + self.invalidate_buffer(0) else: self.error("not found") return diff --git a/Lib/_pyrepl/input.py b/Lib/_pyrepl/input.py index 21c24eb5cde..2d65246c700 100644 --- a/Lib/_pyrepl/input.py +++ b/Lib/_pyrepl/input.py @@ -38,10 +38,11 @@ from abc import ABC, abstractmethod import unicodedata from collections import deque +from typing import TYPE_CHECKING # types -if False: +if TYPE_CHECKING: from .types import EventTuple diff --git a/Lib/_pyrepl/layout.py b/Lib/_pyrepl/layout.py new file mode 100644 index 00000000000..6d854d1142d --- /dev/null +++ b/Lib/_pyrepl/layout.py @@ -0,0 +1,268 @@ +"""Wrap content lines to the terminal width before rendering.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Self + +from .content import ContentFragment, ContentLine +from .types import CursorXY, ScreenInfoRow + + +@dataclass(frozen=True, slots=True) +class LayoutRow: + """Metadata for one physical screen row. + + For the row ``>>> def greet(name):``:: + + >>> def greet(name): + ╰─╯ ╰──────────────╯ + 4 char_widths=(1,1,1,…) ← 16 entries + buffer_advance=17 ← includes the newline + """ + + prompt_width: int + char_widths: tuple[int, ...] + suffix_width: int = 0 + buffer_advance: int = 0 + + @property + def width(self) -> int: + return self.prompt_width + sum(self.char_widths) + self.suffix_width + + @property + def screeninfo(self) -> ScreenInfoRow: + widths = list(self.char_widths) + if self.suffix_width: + widths.append(self.suffix_width) + return self.prompt_width, widths + + +@dataclass(frozen=True, slots=True) +class LayoutMap: + """Mapping between buffer positions and screen coordinates. + + Single source of truth for cursor placement. Given:: + + >>> def greet(name): ← row 0, buffer_advance=17 + ... return name ← row 1, buffer_advance=15 + ▲cursor + + ``pos_to_xy(31)`` → ``(18, 1)``: prompt width 4 + 14 body chars. + """ + rows: tuple[LayoutRow, ...] + + @classmethod + def empty(cls) -> Self: + return cls((LayoutRow(0, ()),)) + + @property + def screeninfo(self) -> list[ScreenInfoRow]: + return [row.screeninfo for row in self.rows] + + def max_column(self, y: int) -> int: + return self.rows[y].width + + def max_row(self) -> int: + return len(self.rows) - 1 + + def pos_to_xy(self, pos: int) -> CursorXY: + if not self.rows: + return 0, 0 + + remaining = pos + for y, row in enumerate(self.rows): + if remaining <= len(row.char_widths): + # Prompt-only leading rows are terminal scenery, not real + # buffer positions. Treating them as real just manufactures + # bugs. + if remaining == 0 and not row.char_widths and row.buffer_advance == 0 and y < len(self.rows) - 1: + continue + x = row.prompt_width + for width in row.char_widths[:remaining]: + x += width + return x, y + remaining -= row.buffer_advance + last_row = self.rows[-1] + return last_row.width - last_row.suffix_width, len(self.rows) - 1 + + def xy_to_pos(self, x: int, y: int) -> int: + if not self.rows: + return 0 + + pos = 0 + for row in self.rows[:y]: + pos += row.buffer_advance + + row = self.rows[y] + cur_x = row.prompt_width + char_widths = row.char_widths + i = 0 + for i, width in enumerate(char_widths): + if cur_x >= x: + # Include trailing zero-width (combining) chars at this position + for trailing_width in char_widths[i:]: + if trailing_width == 0: + pos += 1 + else: + break + return pos + if width == 0: + pos += 1 + continue + cur_x += width + pos += 1 + return pos + + +@dataclass(frozen=True, slots=True) +class WrappedRow: + """One physical screen row after wrapping, ready for rendering. + + When a line overflows the terminal width, it splits into + multiple rows with a ``\\`` continuation marker:: + + >>> x = "a very long li\\ ← suffix="\\", suffix_width=1 + ne that wraps" ← prompt_text="" (continuation) + """ + prompt_text: str = "" + prompt_width: int = 0 + fragments: tuple[ContentFragment, ...] = () + layout_widths: tuple[int, ...] = () + suffix: str = "" + suffix_width: int = 0 + buffer_advance: int = 0 + + +@dataclass(frozen=True, slots=True) +class LayoutResult: + wrapped_rows: tuple[WrappedRow, ...] + layout_map: LayoutMap + line_end_offsets: tuple[int, ...] + + +def layout_content_lines( + lines: tuple[ContentLine, ...], + width: int, + start_offset: int, +) -> LayoutResult: + """Wrap content lines to fit *width* columns. + + A short line passes through as one ``WrappedRow``; a long line is + split at the column boundary with ``\\`` markers:: + + >>> short = 1 ← one WrappedRow + >>> x = "a long stri\\ ← two WrappedRows, first has suffix="\\" + ng" + """ + if width <= 0: + return LayoutResult((), LayoutMap(()), ()) + + offset = start_offset + wrapped_rows: list[WrappedRow] = [] + layout_rows: list[LayoutRow] = [] + line_end_offsets: list[int] = [] + + for line in lines: + newline_advance = int(line.source.has_newline) + for leading in line.prompt.leading_lines: + line_end_offsets.append(offset) + wrapped_rows.append( + WrappedRow( + fragments=(leading,), + ) + ) + layout_rows.append(LayoutRow(0, (), buffer_advance=0)) + + prompt_text = line.prompt.text + prompt_width = line.prompt.width + body = tuple(line.body) + body_widths = tuple(fragment.width for fragment in body) + + # Fast path: line fits on one row. + if not body_widths or (sum(body_widths) + prompt_width) < width: + offset += len(body) + newline_advance + line_end_offsets.append(offset) + wrapped_rows.append( + WrappedRow( + prompt_text=prompt_text, + prompt_width=prompt_width, + fragments=body, + layout_widths=body_widths, + buffer_advance=len(body) + newline_advance, + ) + ) + layout_rows.append( + LayoutRow( + prompt_width, + body_widths, + buffer_advance=len(body) + newline_advance, + ) + ) + continue + + # Slow path: line needs wrapping. + current_prompt = prompt_text + current_prompt_width = prompt_width + start = 0 + total = len(body) + while True: + # Find how many characters fit on this row. + index_to_wrap_before = 0 + column = 0 + for char_width in body_widths[start:]: + if column + char_width + current_prompt_width >= width: + break + index_to_wrap_before += 1 + column += char_width + + if index_to_wrap_before == 0 and start < total: + index_to_wrap_before = 1 # force progress + + at_line_end = (start + index_to_wrap_before) >= total + if at_line_end: + offset += index_to_wrap_before + newline_advance + suffix = "" + suffix_width = 0 + buffer_advance = index_to_wrap_before + newline_advance + else: + offset += index_to_wrap_before + suffix = "\\" + suffix_width = 1 + buffer_advance = index_to_wrap_before + + end = start + index_to_wrap_before + row_fragments = body[start:end] + row_widths = body_widths[start:end] + line_end_offsets.append(offset) + wrapped_rows.append( + WrappedRow( + prompt_text=current_prompt, + prompt_width=current_prompt_width, + fragments=row_fragments, + layout_widths=row_widths, + suffix=suffix, + suffix_width=suffix_width, + buffer_advance=buffer_advance, + ) + ) + layout_rows.append( + LayoutRow( + current_prompt_width, + row_widths, + suffix_width=suffix_width, + buffer_advance=buffer_advance, + ) + ) + + start = end + current_prompt = "" + current_prompt_width = 0 + if at_line_end: + break + + return LayoutResult( + tuple(wrapped_rows), + LayoutMap(tuple(layout_rows)), + tuple(line_end_offsets), + ) diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py index 1fddc63e3ee..92afaa5933a 100644 --- a/Lib/_pyrepl/pager.py +++ b/Lib/_pyrepl/pager.py @@ -138,7 +138,7 @@ def pipe_pager(text: str, cmd: str, title: str = '') -> None: '.' '?e (END):?pB %pB\\%..' ' (press h for help or q to quit)') - env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string) + env['LESS'] = '-RcmPm{0}$PM{0}$'.format(prompt_string) proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, errors='backslashreplace', env=env) assert proc.stdin is not None diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 0ebd9162eca..7e4dd801c84 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -25,16 +25,41 @@ import _colorize from contextlib import contextmanager -from dataclasses import dataclass, field, fields +from dataclasses import dataclass, field, fields, replace +from typing import Self from . import commands, console, input -from .utils import wlen, unbracket, disp_str, gen_colors, THEME +from .content import ( + ContentFragment, + ContentLine, + SourceLine, + build_body_fragments, + process_prompt as build_prompt_content, +) +from .layout import LayoutMap, LayoutResult, LayoutRow, WrappedRow, layout_content_lines +from .render import RenderCell, RenderLine, RenderedScreen, ScreenOverlay +from .utils import ANSI_ESCAPE_SEQUENCE, ColorSpan, THEME, StyleRef, wlen, gen_colors from .trace import trace # types Command = commands.Command -from .types import Callback, SimpleContextManager, KeySpec, CommandName +from collections.abc import Callable, Iterator +from .types import ( + Callback, + CommandName, + CursorXY, + Dimensions, + EventData, + KeySpec, + Keymap, + ScreenInfoRow, + SimpleContextManager, +) + +type CommandClass = type[Command] +type CommandInput = tuple[CommandName | CommandClass, EventData] +type PromptCellCacheKey = tuple[str, bool] # syntax classes @@ -52,8 +77,8 @@ def make_default_syntax_table() -> dict[str, int]: return st -def make_default_commands() -> dict[CommandName, type[Command]]: - result: dict[CommandName, type[Command]] = {} +def make_default_commands() -> dict[CommandName, CommandClass]: + result: dict[CommandName, CommandClass] = {} for v in vars(commands).values(): if isinstance(v, type) and issubclass(v, Command) and v.__name__[0].islower(): result[v.__name__] = v @@ -61,7 +86,7 @@ def make_default_commands() -> dict[CommandName, type[Command]]: return result -default_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( +default_keymap: Keymap = tuple( [ (r"\C-a", "beginning-of-line"), (r"\C-b", "left"), @@ -131,6 +156,77 @@ def make_default_commands() -> dict[CommandName, type[Command]]: ) +@dataclass(frozen=True, slots=True) +class RefreshInvalidation: + """Which parts of the screen need to be recomputed on the next refresh.""" + + cursor_only: bool = False + buffer_from_pos: int | None = None + prompt: bool = False + layout: bool = False + theme: bool = False + message: bool = False + overlay: bool = False + full: bool = False + + @classmethod + def empty(cls) -> Self: + return cls() + + @property + def needs_screen_refresh(self) -> bool: + return any( + ( + self.buffer_from_pos is not None, + self.prompt, + self.layout, + self.theme, + self.message, + self.overlay, + self.full, + ) + ) + + @property + def is_cursor_only(self) -> bool: + return self.cursor_only and not self.needs_screen_refresh + + @property + def buffer_rebuild_from_pos(self) -> int | None: + if self.full or self.prompt or self.layout or self.theme: + return 0 + return self.buffer_from_pos + + def with_cursor(self) -> Self: + if self.needs_screen_refresh: + return self + return replace(self, cursor_only=True) + + def with_buffer(self, from_pos: int) -> Self: + current = from_pos + if self.buffer_from_pos is not None: + current = min(current, self.buffer_from_pos) + return replace(self, cursor_only=False, buffer_from_pos=current) + + def with_prompt(self) -> Self: + return replace(self, cursor_only=False, prompt=True) + + def with_layout(self) -> Self: + return replace(self, cursor_only=False, layout=True) + + def with_theme(self) -> Self: + return replace(self, cursor_only=False, theme=True) + + def with_message(self) -> Self: + return replace(self, cursor_only=False, message=True) + + def with_overlay(self) -> Self: + return replace(self, cursor_only=False, overlay=True) + + def with_full(self) -> Self: + return replace(self, cursor_only=False, full=True) + + @dataclass(slots=True) class Reader: """The Reader class implements the bare bones of a command reader, @@ -148,10 +244,9 @@ class Reader: * pos: A 0-based index into 'buffer' for where the insertion point is. - * screeninfo: - A list of screen position tuples. Each list element is a tuple - representing information on visible line length for a given line. - Allows for efficient skipping of color escape sequences. + * layout: + A mapping between buffer positions and rendered rows/columns. + It is the internal source of truth for cursor placement. * cxy, lxy: the position of the insertion point in screen ... * syntax_table: @@ -162,8 +257,6 @@ class Reader: * arg: The emacs-style prefix argument. It will be None if no such argument has been provided. - * dirty: - True if we need to refresh the display. * kill_ring: The emacs-style kill-ring; manipulated with yank & yank-pop * ps1, ps2, ps3, ps4: @@ -198,66 +291,89 @@ class Reader: kill_ring: list[list[str]] = field(default_factory=list) msg: str = "" arg: int | None = None - dirty: bool = False finished: bool = False paste_mode: bool = False - commands: dict[str, type[Command]] = field(default_factory=make_default_commands) - last_command: type[Command] | None = None + commands: dict[CommandName, CommandClass] = field(default_factory=make_default_commands) + last_command: CommandClass | None = None syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table) - keymap: tuple[tuple[str, str], ...] = () + keymap: Keymap = () input_trans: input.KeymapTranslator = field(init=False) input_trans_stack: list[input.KeymapTranslator] = field(default_factory=list) - screen: list[str] = field(default_factory=list) - screeninfo: list[tuple[int, list[int]]] = field(init=False) - cxy: tuple[int, int] = field(init=False) - lxy: tuple[int, int] = field(init=False) - scheduled_commands: list[str] = field(default_factory=list) + rendered_screen: RenderedScreen = field(init=False) + layout: LayoutMap = field(init=False) + cxy: CursorXY = field(init=False) + lxy: CursorXY = field(init=False) + scheduled_commands: list[CommandName] = field(default_factory=list) can_colorize: bool = False + gen_colors: Callable[[str], Iterator[ColorSpan]] = gen_colors threading_hook: Callback | None = None + invalidation: RefreshInvalidation = field(init=False) ## cached metadata to speed up screen refreshes @dataclass class RefreshCache: - screen: list[str] = field(default_factory=list) - screeninfo: list[tuple[int, list[int]]] = field(init=False) + """Previously computed render/layout data for incremental refresh.""" + + render_lines: list[RenderLine] = field(default_factory=list) + layout_rows: list[LayoutRow] = field(default_factory=list) line_end_offsets: list[int] = field(default_factory=list) - pos: int = field(init=False) - cxy: tuple[int, int] = field(init=False) - dimensions: tuple[int, int] = field(init=False) - invalidated: bool = False + pos: int = 0 + dimensions: Dimensions = (0, 0) def update_cache(self, reader: Reader, - screen: list[str], - screeninfo: list[tuple[int, list[int]]], + render_lines: list[RenderLine], + layout_rows: list[LayoutRow], + line_end_offsets: list[int], ) -> None: - self.screen = screen.copy() - self.screeninfo = screeninfo.copy() + self.render_lines = render_lines.copy() + self.layout_rows = layout_rows.copy() + self.line_end_offsets = line_end_offsets.copy() self.pos = reader.pos - self.cxy = reader.cxy self.dimensions = reader.console.width, reader.console.height - self.invalidated = False def valid(self, reader: Reader) -> bool: - if self.invalidated: - return False dimensions = reader.console.width, reader.console.height dimensions_changed = dimensions != self.dimensions return not dimensions_changed - def get_cached_location(self, reader: Reader) -> tuple[int, int]: - if self.invalidated: - raise ValueError("Cache is invalidated") - offset = 0 - earliest_common_pos = min(reader.pos, self.pos) + def get_cached_location( + self, + reader: Reader, + buffer_from_pos: int | None = None, + *, + reuse_full: bool = False, + ) -> tuple[int, int]: + """Return (buffer_offset, num_reusable_lines) for incremental refresh. + + Three paths: + - reuse_full (overlay/message-only): reuse all cached lines. + - buffer_from_pos=None (full rebuild): rewind to common cursor pos. + - explicit buffer_from_pos: reuse lines before that position. + """ + if reuse_full: + if self.line_end_offsets: + last_offset = self.line_end_offsets[-1] + if last_offset >= len(reader.buffer): + return last_offset, len(self.line_end_offsets) + return 0, 0 + if buffer_from_pos is None: + buffer_from_pos = min(reader.pos, self.pos) num_common_lines = len(self.line_end_offsets) while num_common_lines > 0: - offset = self.line_end_offsets[num_common_lines - 1] - if earliest_common_pos > offset: + candidate = self.line_end_offsets[num_common_lines - 1] + if buffer_from_pos > candidate: break num_common_lines -= 1 - else: - offset = 0 + # Prompt-only leading rows consume no buffer content. Reusing them + # in isolation causes the next incremental rebuild to emit them a + # second time. + while ( + num_common_lines > 0 + and self.layout_rows[num_common_lines - 1].buffer_advance == 0 + ): + num_common_lines -= 1 + offset = self.line_end_offsets[num_common_lines - 1] if num_common_lines else 0 return offset, num_common_lines last_refresh_cache: RefreshCache = field(default_factory=RefreshCache) @@ -270,123 +386,267 @@ def __post_init__(self) -> None: self.input_trans = input.KeymapTranslator( self.keymap, invalid_cls="invalid-key", character_cls="self-insert" ) - self.screeninfo = [(0, [])] + self.layout = LayoutMap.empty() self.cxy = self.pos2xy() self.lxy = (self.pos, 0) + self.rendered_screen = RenderedScreen.empty() self.can_colorize = _colorize.can_colorize() + self.invalidation = RefreshInvalidation.empty() - self.last_refresh_cache.screeninfo = self.screeninfo + self.last_refresh_cache.layout_rows = list(self.layout.rows) self.last_refresh_cache.pos = self.pos - self.last_refresh_cache.cxy = self.cxy + self.last_refresh_cache.dimensions = (0, 0) - def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + @property + def screen(self) -> list[str]: + return list(self.rendered_screen.screen_lines) + + @property + def screeninfo(self) -> list[ScreenInfoRow]: + return self.layout.screeninfo + + def collect_keymap(self) -> Keymap: return default_keymap - def calc_screen(self) -> list[str]: - """Translate changes in self.buffer into changes in self.console.screen.""" - # Since the last call to calc_screen: - # screen and screeninfo may differ due to a completion menu being shown - # pos and cxy may differ due to edits, cursor movements, or completion menus - - # Lines that are above both the old and new cursor position can't have changed, - # unless the terminal has been resized (which might cause reflowing) or we've - # entered or left paste mode (which changes prompts, causing reflowing). + def calc_screen(self) -> RenderedScreen: + """Translate the editable buffer into a base rendered screen.""" num_common_lines = 0 offset = 0 if self.last_refresh_cache.valid(self): - offset, num_common_lines = self.last_refresh_cache.get_cached_location(self) + if ( + self.invalidation.buffer_from_pos is None + and not ( + self.invalidation.full + or self.invalidation.prompt + or self.invalidation.layout + or self.invalidation.theme + ) + and (self.invalidation.message or self.invalidation.overlay) + ): + # Fast path: only overlays or messages changed. + offset, num_common_lines = self.last_refresh_cache.get_cached_location( + self, + reuse_full=True, + ) + assert not self.last_refresh_cache.line_end_offsets or ( + self.last_refresh_cache.line_end_offsets[-1] >= len(self.buffer) + ), "Buffer modified without invalidate_buffer() call" + else: + offset, num_common_lines = self.last_refresh_cache.get_cached_location( + self, + self._buffer_refresh_from_pos(), + ) - screen = self.last_refresh_cache.screen - del screen[num_common_lines:] + base_render_lines = self.last_refresh_cache.render_lines[:num_common_lines] + layout_rows = self.last_refresh_cache.layout_rows[:num_common_lines] + last_refresh_line_end_offsets = self.last_refresh_cache.line_end_offsets[:num_common_lines] - screeninfo = self.last_refresh_cache.screeninfo - del screeninfo[num_common_lines:] + source_lines = self._build_source_lines(offset, num_common_lines) + content_lines = self._build_content_lines( + source_lines, + prompt_from_cache=bool(offset and self.buffer[offset - 1] != "\n"), + ) + layout_result = self._layout_content(content_lines, offset) + base_render_lines.extend(self._render_wrapped_rows(layout_result.wrapped_rows)) + layout_rows.extend(layout_result.layout_map.rows) + last_refresh_line_end_offsets.extend(layout_result.line_end_offsets) - last_refresh_line_end_offsets = self.last_refresh_cache.line_end_offsets - del last_refresh_line_end_offsets[num_common_lines:] + self.layout = LayoutMap(tuple(layout_rows)) + self.cxy = self.pos2xy() + if not source_lines: + # reuse_full path: _build_source_lines didn't run, + # so lxy wasn't updated. Derive it from the buffer. + self.lxy = self._compute_lxy() + self.last_refresh_cache.update_cache( + self, + base_render_lines, + layout_rows, + last_refresh_line_end_offsets, + ) + return RenderedScreen(tuple(base_render_lines), self.cxy) - pos = self.pos - pos -= offset + def _buffer_refresh_from_pos(self) -> int: + """Return buffer position from which to rebuild content. - prompt_from_cache = (offset and self.buffer[offset - 1] != "\n") + Returns 0 (full rebuild) when no incremental position is known. + """ + buffer_from_pos = self.invalidation.buffer_rebuild_from_pos + if buffer_from_pos is not None: + return buffer_from_pos + return 0 - if self.can_colorize: - colors = list(gen_colors(self.get_unicode())) + def _compute_lxy(self) -> CursorXY: + """Derive logical cursor (col, lineno) from the buffer and pos.""" + text = "".join(self.buffer[:self.pos]) + lineno = text.count("\n") + if lineno: + col = self.pos - text.rindex("\n") - 1 else: - colors = None - trace("colors = {colors}", colors=colors) + col = self.pos + return col, lineno + + def _build_source_lines( + self, + offset: int, + first_lineno: int, + ) -> tuple[SourceLine, ...]: + if offset == len(self.buffer) and (offset > 0 or first_lineno > 0): + return () + + pos = self.pos - offset lines = "".join(self.buffer[offset:]).split("\n") cursor_found = False lines_beyond_cursor = 0 - for ln, line in enumerate(lines, num_common_lines): + source_lines: list[SourceLine] = [] + current_offset = offset + + for line_index, line in enumerate(lines): + lineno = first_lineno + line_index + has_newline = line_index < len(lines) - 1 line_len = len(line) + cursor_index: int | None = None if 0 <= pos <= line_len: - self.lxy = pos, ln + cursor_index = pos + self.lxy = pos, lineno cursor_found = True elif cursor_found: lines_beyond_cursor += 1 if lines_beyond_cursor > self.console.height: - # No need to keep formatting lines. - # The console can't show them. break + + source_lines.append( + SourceLine( + lineno=lineno, + text=line, + start_offset=current_offset, + has_newline=has_newline, + cursor_index=cursor_index, + ) + ) + pos -= line_len + 1 + current_offset += line_len + (1 if has_newline else 0) + + return tuple(source_lines) + + def _build_content_lines( + self, + source_lines: tuple[SourceLine, ...], + *, + prompt_from_cache: bool, + ) -> tuple[ContentLine, ...]: + if self.can_colorize: + colors = list(self.gen_colors(self.get_unicode())) + else: + colors = None + trace("colors = {colors}", colors=colors) + + content_lines: list[ContentLine] = [] + for source_line in source_lines: if prompt_from_cache: - # Only the first line's prompt can come from the cache prompt_from_cache = False prompt = "" else: - prompt = self.get_prompt(ln, line_len >= pos >= 0) - while "\n" in prompt: - pre_prompt, _, prompt = prompt.partition("\n") - last_refresh_line_end_offsets.append(offset) - screen.append(pre_prompt) - screeninfo.append((0, [])) - pos -= line_len + 1 - prompt, prompt_len = self.process_prompt(prompt) - chars, char_widths = disp_str(line, colors, offset) - wrapcount = (sum(char_widths) + prompt_len) // self.console.width - if wrapcount == 0 or not char_widths: - offset += line_len + 1 # Takes all of the line plus the newline - last_refresh_line_end_offsets.append(offset) - screen.append(prompt + "".join(chars)) - screeninfo.append((prompt_len, char_widths)) - else: - pre = prompt - prelen = prompt_len - for wrap in range(wrapcount + 1): - index_to_wrap_before = 0 - column = 0 - for char_width in char_widths: - if column + char_width + prelen >= self.console.width: - break - index_to_wrap_before += 1 - column += char_width - if len(chars) > index_to_wrap_before: - offset += index_to_wrap_before - post = "\\" - after = [1] - else: - offset += index_to_wrap_before + 1 # Takes the newline - post = "" - after = [] - last_refresh_line_end_offsets.append(offset) - render = pre + "".join(chars[:index_to_wrap_before]) + post - render_widths = char_widths[:index_to_wrap_before] + after - screen.append(render) - screeninfo.append((prelen, render_widths)) - chars = chars[index_to_wrap_before:] - char_widths = char_widths[index_to_wrap_before:] - pre = "" - prelen = 0 - self.screeninfo = screeninfo - self.cxy = self.pos2xy() - if self.msg: - for mline in self.msg.split("\n"): - screen.append(mline) - screeninfo.append((0, [])) + prompt = self.get_prompt(source_line.lineno, source_line.cursor_on_line) + content_lines.append( + ContentLine( + source=source_line, + prompt=build_prompt_content(prompt), + body=build_body_fragments( + source_line.text, + colors, + source_line.start_offset, + ), + ) + ) + return tuple(content_lines) - self.last_refresh_cache.update_cache(self, screen, screeninfo) - return screen + def _layout_content( + self, + content_lines: tuple[ContentLine, ...], + offset: int, + ) -> LayoutResult: + return layout_content_lines(content_lines, self.console.width, offset) + + def _render_wrapped_rows( + self, + wrapped_rows: tuple[WrappedRow, ...], + ) -> list[RenderLine]: + return [ + self._render_line( + row.prompt_text, + row.fragments, + row.suffix, + ) + for row in wrapped_rows + ] + + def _render_message_lines(self) -> tuple[RenderLine, ...]: + if not self.msg: + return () + width = self.console.width + render_lines: list[RenderLine] = [] + for message_line in self.msg.split("\n"): + # If self.msg is larger than console width, make it fit. + # TODO: try to split between words? + if not message_line: + render_lines.append(RenderLine.from_rendered_text("")) + continue + for offset in range(0, len(message_line), width): + render_lines.append( + RenderLine.from_rendered_text(message_line[offset : offset + width]) + ) + return tuple(render_lines) + + def get_screen_overlays(self) -> tuple[ScreenOverlay, ...]: + return () + + def compose_rendered_screen(self, base_screen: RenderedScreen) -> RenderedScreen: + overlays = list(self.get_screen_overlays()) + message_lines = self._render_message_lines() + if message_lines: + overlays.append(ScreenOverlay(len(base_screen.lines), message_lines)) + if not overlays: + return base_screen + return RenderedScreen(base_screen.lines, base_screen.cursor, tuple(overlays)) + + _prompt_cell_cache: dict[PromptCellCacheKey, tuple[RenderCell, ...]] = field( + init=False, default_factory=dict, repr=False + ) + + def _render_line( + self, + prefix: str, + fragments: tuple[ContentFragment, ...], + suffix: str = "", + ) -> RenderLine: + cells: list[RenderCell] = [] + if prefix: + cache_key = (prefix, self.can_colorize) + cached = self._prompt_cell_cache.get(cache_key) + if cached is None: + prompt_cells = RenderLine.from_rendered_text(prefix).cells + if self.can_colorize and prompt_cells and not ANSI_ESCAPE_SEQUENCE.search(prefix): + prompt_style = StyleRef.from_tag("prompt", THEME()["prompt"]) + prompt_cells = tuple( + RenderCell( + cell.text, + cell.width, + style=prompt_style if cell.text else cell.style, + controls=cell.controls, + ) + for cell in prompt_cells + ) + self._prompt_cell_cache[cache_key] = prompt_cells + cached = prompt_cells + cells.extend(cached) + cells.extend( + RenderCell(fragment.text, fragment.width, style=fragment.style) + for fragment in fragments + ) + if suffix: + cells.extend(RenderLine.from_rendered_text(suffix).cells) + return RenderLine.from_cells(cells) @staticmethod def process_prompt(prompt: str) -> tuple[str, int]: @@ -396,9 +656,8 @@ def process_prompt(prompt: str) -> tuple[str, int]: (\x01 and \x02) removed. The length ignores anything between those brackets as well as any ANSI escape sequences. """ - out_prompt = unbracket(prompt, including_content=False) - visible_prompt = unbracket(prompt, including_content=True) - return out_prompt, wlen(visible_prompt) + prompt_content = build_prompt_content(prompt) + return prompt_content.text, prompt_content.width def bow(self, p: int | None = None) -> int: """Return the 0-based index of the word break preceding p most @@ -460,10 +719,10 @@ def eol(self, p: int | None = None) -> int: def max_column(self, y: int) -> int: """Return the last x-offset for line y""" - return self.screeninfo[y][0] + sum(self.screeninfo[y][1]) + return self.layout.max_column(y) def max_row(self) -> int: - return len(self.screeninfo) - 1 + return self.layout.max_row() def get_arg(self, default: int = 1) -> int: """Return any prefix argument that the user has supplied, @@ -489,10 +748,6 @@ def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: prompt = self.ps3 else: prompt = self.ps1 - - if self.can_colorize: - t = THEME() - prompt = f"{t.prompt}{prompt}{t.reset}" return prompt def push_input_trans(self, itrans: input.KeymapTranslator) -> None: @@ -504,65 +759,48 @@ def pop_input_trans(self) -> None: def setpos_from_xy(self, x: int, y: int) -> None: """Set pos according to coordinates x, y""" - pos = 0 - i = 0 - while i < y: - prompt_len, char_widths = self.screeninfo[i] - offset = len(char_widths) - in_wrapped_line = prompt_len + sum(char_widths) >= self.console.width - if in_wrapped_line: - pos += offset - 1 # -1 cause backslash is not in buffer - else: - pos += offset + 1 # +1 cause newline is in buffer - i += 1 + self.pos = self.layout.xy_to_pos(x, y) - j = 0 - cur_x = self.screeninfo[i][0] - while cur_x < x: - if self.screeninfo[i][1][j] == 0: - j += 1 # prevent potential future infinite loop - continue - cur_x += self.screeninfo[i][1][j] - j += 1 - pos += 1 - - self.pos = pos - - def pos2xy(self) -> tuple[int, int]: + def pos2xy(self) -> CursorXY: """Return the x, y coordinates of position 'pos'.""" - - prompt_len, y = 0, 0 - char_widths: list[int] = [] - pos = self.pos - assert 0 <= pos <= len(self.buffer) - - # optimize for the common case: typing at the end of the buffer - if pos == len(self.buffer) and len(self.screeninfo) > 0: - y = len(self.screeninfo) - 1 - prompt_len, char_widths = self.screeninfo[y] - return prompt_len + sum(char_widths), y - - for prompt_len, char_widths in self.screeninfo: - offset = len(char_widths) - in_wrapped_line = prompt_len + sum(char_widths) >= self.console.width - if in_wrapped_line: - offset -= 1 # need to remove line-wrapping backslash - - if offset >= pos: - break - - if not in_wrapped_line: - offset += 1 # there's a newline in buffer - - pos -= offset - y += 1 - return prompt_len + sum(char_widths[:pos]), y + assert 0 <= self.pos <= len(self.buffer) + return self.layout.pos_to_xy(self.pos) def insert(self, text: str | list[str]) -> None: """Insert 'text' at the insertion point.""" + start = self.pos self.buffer[self.pos : self.pos] = list(text) self.pos += len(text) - self.dirty = True + self.invalidate_buffer(start) + + def invalidate_cursor(self) -> None: + self.invalidation = self.invalidation.with_cursor() + + def invalidate_buffer(self, from_pos: int) -> None: + self.invalidation = self.invalidation.with_buffer(from_pos) + + def invalidate_prompt(self) -> None: + self._prompt_cell_cache.clear() + self.invalidation = self.invalidation.with_prompt() + + def invalidate_layout(self) -> None: + self.invalidation = self.invalidation.with_layout() + + def invalidate_theme(self) -> None: + self._prompt_cell_cache.clear() + self.invalidation = self.invalidation.with_theme() + + def invalidate_message(self) -> None: + self.invalidation = self.invalidation.with_message() + + def invalidate_overlay(self) -> None: + self.invalidation = self.invalidation.with_overlay() + + def invalidate_full(self) -> None: + self.invalidation = self.invalidation.with_full() + + def clear_invalidation(self) -> None: + self.invalidation = RefreshInvalidation.empty() def update_cursor(self) -> None: """Move the cursor to reflect changes in self.pos""" @@ -574,7 +812,7 @@ def after_command(self, cmd: Command) -> None: """This function is called to allow post command cleanup.""" if getattr(cmd, "kills_digit_arg", True): if self.arg is not None: - self.dirty = True + self.invalidate_prompt() self.arg = None def prepare(self) -> None: @@ -587,9 +825,15 @@ def prepare(self) -> None: self.finished = False del self.buffer[:] self.pos = 0 - self.dirty = True + self.layout = LayoutMap.empty() + self.cxy = self.pos2xy() + self.lxy = (self.pos, 0) + self.rendered_screen = RenderedScreen.empty() + self.invalidate_full() self.last_command = None - self.calc_screen() + base_screen = self.calc_screen() + self.rendered_screen = self.compose_rendered_screen(base_screen) + self.invalidation = RefreshInvalidation.empty() except BaseException: self.restore() raise @@ -598,7 +842,7 @@ def prepare(self) -> None: cmd = self.scheduled_commands.pop() self.do_cmd((cmd, [])) - def last_command_is(self, cls: type) -> bool: + def last_command_is(self, cls: CommandClass) -> bool: if not self.last_command: return False return issubclass(cls, self.last_command) @@ -619,27 +863,51 @@ def suspend(self) -> SimpleContextManager: setattr(self, arg, prev_state[arg]) self.prepare() + @contextmanager + def suspend_colorization(self) -> SimpleContextManager: + try: + old_can_colorize = self.can_colorize + self.can_colorize = False + yield + finally: + self.can_colorize = old_can_colorize + def finish(self) -> None: """Called when a command signals that we're finished.""" pass def error(self, msg: str = "none") -> None: self.msg = "! " + msg + " " - self.dirty = True + self.invalidate_message() self.console.beep() def update_screen(self) -> None: - if self.dirty: + if self.invalidation.is_cursor_only: + self.update_cursor() + self.clear_invalidation() + elif self.invalidation.needs_screen_refresh: self.refresh() def refresh(self) -> None: """Recalculate and refresh the screen.""" + self.console.height, self.console.width = self.console.getheightwidth() # this call sets up self.cxy, so call it first. - self.screen = self.calc_screen() - self.console.refresh(self.screen, self.cxy) - self.dirty = False + base_screen = self.calc_screen() + rendered_screen = self.compose_rendered_screen(base_screen) + self.rendered_screen = rendered_screen + trace( + "reader.refresh cursor={cursor} lines={lines} " + "dims=({width},{height}) invalidation={invalidation}", + cursor=self.cxy, + lines=len(rendered_screen.composed_lines), + width=self.console.width, + height=self.console.height, + invalidation=self.invalidation, + ) + self.console.refresh(rendered_screen) + self.clear_invalidation() - def do_cmd(self, cmd: tuple[str, list[str]]) -> None: + def do_cmd(self, cmd: CommandInput) -> None: """`cmd` is a tuple of "event_name" and "event", which in the current implementation is always just the "buffer" which happens to be a list of single-character strings.""" @@ -656,13 +924,14 @@ def do_cmd(self, cmd: tuple[str, list[str]]) -> None: command.do() self.after_command(command) + if ( + not self.invalidation.needs_screen_refresh + and not self.invalidation.is_cursor_only + ): + self.invalidate_cursor() + self.update_screen() - if self.dirty: - self.refresh() - else: - self.update_cursor() - - if not isinstance(cmd, commands.digit_arg): + if command_type is not commands.digit_arg: self.last_command = command_type self.finished = bool(command.finish) @@ -695,7 +964,7 @@ def handle1(self, block: bool = True) -> bool: if self.msg: self.msg = "" - self.dirty = True + self.invalidate_message() while True: # We use the same timeout as in readline.c: 100ms @@ -712,9 +981,13 @@ def handle1(self, block: bool = True) -> bool: if event.evt == "key": self.input_trans.push(event) elif event.evt == "scroll": + self.invalidate_full() self.refresh() + return True elif event.evt == "resize": + self.invalidate_full() self.refresh() + return True else: translate = False diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 23b8fa6b9c7..f8f1727d2a1 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -37,9 +37,10 @@ from rlcompleter import Completer as RLCompleter from . import commands, historical_reader -from .completing_reader import CompletingReader +from .completing_reader import CompletingReader, stripcolor from .console import Console as ConsoleType from ._module_completer import ModuleCompleter, make_default_module_completer +from .fancycompleter import Completer as FancyCompleter Console: type[ConsoleType] _error: tuple[type[Exception], ...] | type[Exception] @@ -55,7 +56,7 @@ # types Command = commands.Command from collections.abc import Callable, Collection -from .types import Callback, Completer, KeySpec, CommandName +from .types import Callback, Completer, KeySpec, CommandName, CompletionAction TYPE_CHECKING = False @@ -134,7 +135,7 @@ def get_stem(self) -> str: p -= 1 return "".join(b[p + 1 : self.pos]) - def get_completions(self, stem: str) -> list[str]: + def get_completions(self, stem: str) -> tuple[list[str], CompletionAction | None]: module_completions = self.get_module_completions() if module_completions is not None: return module_completions @@ -144,7 +145,7 @@ def get_completions(self, stem: str) -> list[str]: while p > 0 and b[p - 1] != "\n": p -= 1 num_spaces = 4 - ((self.pos - p) % 4) - return [" " * num_spaces] + return [" " * num_spaces], None result = [] function = self.config.readline_completer if function is not None: @@ -162,12 +163,12 @@ def get_completions(self, stem: str) -> list[str]: break result.append(next) state += 1 - # emulate the behavior of the standard readline that sorts - # the completions before displaying them. - result.sort() - return result + # Emulate readline's sorting using the visible text rather than + # the raw ANSI escape sequences used for colorized matches. + result.sort(key=stripcolor) + return result, None - def get_module_completions(self) -> list[str] | None: + def get_module_completions(self) -> tuple[list[str], CompletionAction | None] | None: line = self.get_line() return self.config.module_completer.get_completions(line) @@ -276,7 +277,7 @@ class maybe_accept(commands.Command): def do(self) -> None: r: ReadlineAlikeReader r = self.reader # type: ignore[assignment] - r.dirty = True # this is needed to hide the completion menu, if visible + r.invalidate_overlay() # hide completion menu, if visible # if there are already several lines and the cursor # is not on the last one, always insert a new \n. @@ -336,7 +337,7 @@ def do(self) -> None: break r.pos -= repeat del b[r.pos : r.pos + repeat] - r.dirty = True + r.invalidate_buffer(r.pos) else: self.reader.error("can't backspace at start") @@ -412,8 +413,12 @@ def set_completer_delims(self, delimiters: Collection[str]) -> None: def get_completer_delims(self) -> str: return "".join(sorted(self.config.completer_delims)) - def _histline(self, line: str) -> str: + def _histline(self, line: str, *, sanitize_nuls: bool = False) -> str: line = line.rstrip("\n") + if "\0" in line: + if not sanitize_nuls: + raise ValueError("embedded null character") + line = line.replace("\0", "") return line def get_history_length(self) -> int: @@ -446,9 +451,12 @@ def read_history_file(self, filename: str = gethistoryfile()) -> None: if line.endswith("\r"): buffer.append(line+'\n') else: - line = self._histline(line) + line = self._histline(line, sanitize_nuls=True) if buffer: - line = self._histline("".join(buffer).replace("\r", "") + line) + line = self._histline( + "".join(buffer).replace("\r", "") + line, + sanitize_nuls=True, + ) del buffer[:] if line: history.append(line) @@ -609,7 +617,12 @@ def _setup(namespace: Mapping[str, Any]) -> None: if not isinstance(namespace, dict): namespace = dict(namespace) _wrapper.config.module_completer = ModuleCompleter(namespace) - _wrapper.config.readline_completer = RLCompleter(namespace).complete + use_basic_completer = ( + not sys.flags.ignore_environment + and os.getenv("PYTHON_BASIC_COMPLETER") + ) + completer_cls = RLCompleter if use_basic_completer else FancyCompleter + _wrapper.config.readline_completer = completer_cls(namespace).complete # this is not really what readline.c does. Better than nothing I guess import builtins diff --git a/Lib/_pyrepl/render.py b/Lib/_pyrepl/render.py new file mode 100644 index 00000000000..b821f35d850 --- /dev/null +++ b/Lib/_pyrepl/render.py @@ -0,0 +1,397 @@ +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from dataclasses import dataclass, field +from typing import Literal, Protocol, Self + +from .utils import ANSI_ESCAPE_SEQUENCE, THEME, StyleRef, str_width +from .types import CursorXY + +type RenderStyle = StyleRef | str | None +type LineUpdateKind = Literal[ + "insert_char", + "replace_char", + "replace_span", + "delete_then_insert", + "rewrite_suffix", +] + + +class _ThemeSyntax(Protocol): + """Protocol for theme objects that map tag names to SGR escape strings.""" + def __getitem__(self, key: str, /) -> str: ... + + +@dataclass(frozen=True, slots=True) +class RenderCell: + """One terminal cell: a character, its column width, and SGR style. + + A screen row like ``>>> def`` is a sequence of cells:: + + > > > d e f + ╰─╯╰─╯╰─╯╰─╯╰─╯╰─╯╰─╯ + """ + + text: str + width: int + style: StyleRef = field(default_factory=StyleRef) + controls: tuple[str, ...] = () + + @property + def terminal_text(self) -> str: + return render_cells((self,)) + + +def _theme_style(theme: _ThemeSyntax, tag: str) -> str: + return theme[tag] + + +def _style_escape(style: StyleRef) -> str: + if style.sgr: + return style.sgr + if style.tag is None: + return "" + return _theme_style(THEME(), style.tag) + + +def _update_terminal_state(state: str, escape: str) -> str: + if escape in {"\x1b[0m", "\x1b[m"}: + return "" + return state + escape + + +def _cells_from_rendered_text(text: str) -> tuple[RenderCell, ...]: + if not text: + return () + + cells: list[RenderCell] = [] + pending_controls: list[str] = [] + active_sgr = "" + index = 0 + + def append_plain_text(segment: str) -> None: + nonlocal pending_controls + if not segment: + return + if pending_controls: + cells.append(RenderCell("", 0, controls=tuple(pending_controls))) + pending_controls = [] + for char in segment: + cells.append( + RenderCell( + char, + str_width(char), + style=StyleRef.from_sgr(active_sgr), + ) + ) + + for match in ANSI_ESCAPE_SEQUENCE.finditer(text): + append_plain_text(text[index : match.start()]) + escape = match.group(0) + if escape.endswith("m"): + active_sgr = _update_terminal_state(active_sgr, escape) + else: + pending_controls.append(escape) + index = match.end() + + append_plain_text(text[index:]) + if pending_controls: + cells.append(RenderCell("", 0, controls=tuple(pending_controls))) + + return tuple(cells) + + +@dataclass(frozen=True, slots=True) +class RenderLine: + """One physical screen row as a tuple of :class:`RenderCell` objects. + + ``text`` is the pre-rendered terminal string (characters + SGR escapes); + ``width`` is the total visible column count. + """ + + cells: tuple[RenderCell, ...] + text: str + width: int + + @classmethod + def from_cells(cls, cells: Iterable[RenderCell]) -> Self: + cell_tuple = tuple(cells) + return cls( + cells=cell_tuple, + text=render_cells(cell_tuple), + width=sum(cell.width for cell in cell_tuple), + ) + + @classmethod + def from_parts( + cls, + parts: Sequence[str], + widths: Sequence[int], + styles: Sequence[RenderStyle] | None = None, + ) -> Self: + if styles is None: + return cls.from_cells( + RenderCell(text, width) + for text, width in zip(parts, widths) + ) + + cells: list[RenderCell] = [] + for text, width, style in zip(parts, widths, styles): + if isinstance(style, StyleRef): + cells.append(RenderCell(text, width, style=style)) + elif style is None: + cells.append(RenderCell(text, width)) + else: + cells.append(RenderCell(text, width, style=StyleRef.from_tag(style))) + return cls.from_cells(cells) + + @classmethod + def from_rendered_text(cls, text: str) -> Self: + return cls.from_cells(_cells_from_rendered_text(text)) + + +@dataclass(frozen=True, slots=True) +class ScreenOverlay: + """An overlay that replaces or inserts lines at a screen position. + + If *insert* is True, lines are spliced in (shifting content down); + if False (default), lines replace existing content at *y*. + + Overlays are used to display tab completion menus and status messages. + For example, a tab-completion menu inserted below the input:: + + >>> os.path.j ← line 0 (base content) + join ← ScreenOverlay(y=1, insert=True) + junction ← (pushes remaining lines down) + ... ← line 1 (shifted down by 2) + """ + y: int + lines: tuple[RenderLine, ...] + insert: bool = False + + +@dataclass(frozen=True, slots=True) +class RenderedScreen: + """The complete screen state: content lines, cursor, and overlays. + + ``lines`` holds the base content; ``composed_lines`` is the final + result after overlays (completion menus, messages) are applied:: + + lines: composed_lines: + ┌──────────────────┐ ┌──────────────────┐ + │>>> os.path.j │ │>>> os.path.j │ + │... │ ──► │ join │ ← overlay + └──────────────────┘ │... │ + └──────────────────┘ + """ + + lines: tuple[RenderLine, ...] + cursor: CursorXY + overlays: tuple[ScreenOverlay, ...] = () + composed_lines: tuple[RenderLine, ...] = field(init=False, default=()) + + def __post_init__(self) -> None: + object.__setattr__(self, "composed_lines", self._compose()) + + def _compose(self) -> tuple[RenderLine, ...]: + """Apply overlays in tuple order; inserts shift subsequent positions.""" + if not self.overlays: + return self.lines + + lines = list(self.lines) + y_offset = 0 + for overlay in self.overlays: + adjusted_y = overlay.y + y_offset + assert adjusted_y >= 0, ( + f"Overlay y={overlay.y} with offset={y_offset} is negative; " + "overlays must be sorted by ascending y" + ) + if overlay.insert: + # Splice overlay lines in, pushing existing content down. + lines[adjusted_y:adjusted_y] = overlay.lines + y_offset += len(overlay.lines) + else: + # Replace existing lines at the overlay position. + target_len = adjusted_y + len(overlay.lines) + if len(lines) < target_len: + lines.extend([EMPTY_RENDER_LINE] * (target_len - len(lines))) + for index, line in enumerate(overlay.lines): + lines[adjusted_y + index] = line + return tuple(lines) + + @classmethod + def empty(cls) -> Self: + return cls((), (0, 0), ()) + + @classmethod + def from_screen_lines( + cls, + screen: Sequence[str], + cursor: CursorXY, + ) -> Self: + return cls( + tuple(RenderLine.from_rendered_text(line) for line in screen), + cursor, + (), + ) + + def with_overlay( + self, + y: int, + lines: Iterable[RenderLine], + ) -> Self: + return type(self)( + self.lines, + self.cursor, + self.overlays + (ScreenOverlay(y, tuple(lines)),), + ) + + @property + def screen_lines(self) -> tuple[str, ...]: + return tuple(line.text for line in self.composed_lines) + + +@dataclass(frozen=True, slots=True) +class LineDiff: + """The changed region between an old and new version of one screen row. + + When the user types ``e`` so the row changes from + ``>>> nam`` to ``>>> name``:: + + >>> n a m old + >>> n a m e new + ╰─╯ + start_cell=7, new_cells=("m","e"), old_cells=("m",) + """ + + start_cell: int + start_x: int + old_cells: tuple[RenderCell, ...] + new_cells: tuple[RenderCell, ...] + old_width: int + new_width: int + + @property + def old_text(self) -> str: + return render_cells(self.old_cells) + + @property + def new_text(self) -> str: + return render_cells(self.new_cells) + + @property + def old_changed_width(self) -> int: + return sum(cell.width for cell in self.old_cells) + + @property + def new_changed_width(self) -> int: + return sum(cell.width for cell in self.new_cells) + + +EMPTY_RENDER_LINE = RenderLine(cells=(), text="", width=0) + + +@dataclass(frozen=True, slots=True) +class LineUpdate: + kind: LineUpdateKind + y: int + start_cell: int + start_x: int + """Screen x-coordinate where the update begins. Used for cursor positioning.""" + cells: tuple[RenderCell, ...] + char_width: int = 0 + clear_eol: bool = False + reset_to_margin: bool = False + """If True, the console must resync the cursor position after writing + (needed when cells contain non-SGR escape sequences that may move the cursor).""" + text: str = field(init=False, default="") + + def __post_init__(self) -> None: + object.__setattr__(self, "text", render_cells(self.cells)) + + +def _controls_require_cursor_resync(controls: Sequence[str]) -> bool: + # Anything beyond SGR means the cursor may no longer be where we left it. + return any(not control.endswith("m") for control in controls) + + +def requires_cursor_resync(cells: Sequence[RenderCell]) -> bool: + return any(_controls_require_cursor_resync(cell.controls) for cell in cells) + + +def render_cells( + cells: Sequence[RenderCell], + visual_style: str | None = None, +) -> str: + """Render a sequence of cells into a terminal string with SGR escapes. + + Tracks the active SGR state to emit resets only when the style + actually changes, minimizing output bytes. + + If *visual_style* is given (used by redraw visualization), it is appended + to every cell's style. + """ + rendered: list[str] = [] + active_escape = "" + for cell in cells: + if cell.controls: + rendered.extend(cell.controls) + if not cell.text: + continue + + target_escape = _style_escape(cell.style) + if visual_style is not None: + target_escape += visual_style + if target_escape != active_escape: + if active_escape: + rendered.append("\x1b[0m") + if target_escape: + rendered.append(target_escape) + active_escape = target_escape + rendered.append(cell.text) + + if active_escape: + rendered.append("\x1b[0m") + return "".join(rendered) + + +def diff_render_lines(old: RenderLine, new: RenderLine) -> LineDiff | None: + if old == new: + return None + + prefix = 0 + start_x = 0 + max_prefix = min(len(old.cells), len(new.cells)) + while prefix < max_prefix and old.cells[prefix] == new.cells[prefix]: + # Stop at any cell with non-SGR controls, since those might affect + # cursor position and must be re-emitted. + if old.cells[prefix].controls: + break + start_x += old.cells[prefix].width + prefix += 1 + + old_suffix = len(old.cells) + new_suffix = len(new.cells) + while old_suffix > prefix and new_suffix > prefix: + old_cell = old.cells[old_suffix - 1] + new_cell = new.cells[new_suffix - 1] + if old_cell.controls or new_cell.controls or old_cell != new_cell: + break + old_suffix -= 1 + new_suffix -= 1 + + # Extend diff range to include trailing zero-width combining characters, + # so we never render a combining char without its base character. + while old_suffix < len(old.cells) and old.cells[old_suffix].width == 0: + old_suffix += 1 + while new_suffix < len(new.cells) and new.cells[new_suffix].width == 0: + new_suffix += 1 + + return LineDiff( + start_cell=prefix, + start_x=start_x, + old_cells=old.cells[prefix:old_suffix], + new_cells=new.cells[prefix:new_suffix], + old_width=old.width, + new_width=new.width, + ) diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index ff1bdab9fea..c169d0191bd 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -31,7 +31,6 @@ import sys import code import warnings -import errno from .readline import _get_reader, multiline_input, append_history_file @@ -125,7 +124,7 @@ def maybe_run_command(statement: str) -> bool: command = REPL_COMMANDS[statement] if callable(command): # Make sure that history does not change because of commands - with reader.suspend_history(): + with reader.suspend_history(), reader.suspend_colorization(): command() return True return False @@ -162,7 +161,7 @@ def maybe_run_command(statement: str) -> bool: if r.input_trans is r.isearch_trans: r.do_cmd(("isearch-end", [""])) r.pos = len(r.get_unicode()) - r.dirty = True + r.invalidate_full() r.refresh() console.write("\nKeyboardInterrupt\n") console.resetbuffer() diff --git a/Lib/_pyrepl/trace.py b/Lib/_pyrepl/trace.py index 943ee12f964..39586780519 100644 --- a/Lib/_pyrepl/trace.py +++ b/Lib/_pyrepl/trace.py @@ -32,3 +32,9 @@ def trace(line: str, *k: object, **kw: object) -> None: line = line.format(*k, **kw) trace_file.write(line + "\n") trace_file.flush() + + +def trace_text(text: str, limit: int = 60) -> str: + if len(text) > limit: + text = text[:limit] + "..." + return repr(text) diff --git a/Lib/_pyrepl/types.py b/Lib/_pyrepl/types.py index c5b7ebc1a40..919763158eb 100644 --- a/Lib/_pyrepl/types.py +++ b/Lib/_pyrepl/types.py @@ -4,7 +4,13 @@ type SimpleContextManager = Iterator[None] type KeySpec = str # like r"\C-c" type CommandName = str # like "interrupt" -type EventTuple = tuple[CommandName, str] +type EventData = list[str] +type EventTuple = tuple[CommandName, EventData] +type CursorXY = tuple[int, int] +type Dimensions = tuple[int, int] +type ScreenInfoRow = tuple[int, list[int]] +type Keymap = tuple[tuple[KeySpec, CommandName], ...] type Completer = Callable[[str, int], str | None] type CharBuffer = list[str] type CharWidths = list[int] +type CompletionAction = tuple[str, Callable[[], str | None]] diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index a7e49923191..9c4644db53e 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -31,14 +31,25 @@ import time import types import platform +from collections.abc import Callable +from dataclasses import dataclass from fcntl import ioctl +from typing import TYPE_CHECKING, cast, overload from . import terminfo from .console import Console, Event -from .fancy_termios import tcgetattr, tcsetattr -from .trace import trace +from .fancy_termios import tcgetattr, tcsetattr, TermState +from .render import ( + EMPTY_RENDER_LINE, + LineUpdate, + RenderLine, + RenderedScreen, + requires_cursor_resync, + diff_render_lines, + render_cells, +) +from .trace import trace, trace_text from .unix_eventqueue import EventQueue -from .utils import wlen # declare posix optional to allow None assignment on other platforms posix: types.ModuleType | None @@ -47,20 +58,21 @@ except ImportError: posix = None -TYPE_CHECKING = False - # types if TYPE_CHECKING: - from typing import IO, Literal, overload -else: - overload = lambda func: None + from typing import AbstractSet, IO, Literal + +type _MoveFunc = Callable[[int, int], None] +type _PendingWrite = tuple[str | bytes, bool] class InvalidTerminal(RuntimeError): - pass + def __init__(self, message: str) -> None: + super().__init__(errno.EIO, message) _error = (termios.error, InvalidTerminal) +_error_codes_to_ignore = frozenset([errno.EIO, errno.ENXIO, errno.EPERM]) SIGWINCH_EVENT = "repaint" @@ -125,18 +137,59 @@ def __init__(self): def register(self, fd, flag): self.fd = fd + # note: The 'timeout' argument is received as *milliseconds* def poll(self, timeout: float | None = None) -> list[int]: if timeout is None: r, w, e = select.select([self.fd], [], []) else: - r, w, e = select.select([self.fd], [], [], timeout/1000) + r, w, e = select.select([self.fd], [], [], timeout / 1000) return r poll = MinimalPoll # type: ignore[assignment] +@dataclass(frozen=True, slots=True) +class UnixRefreshPlan: + """Instructions for updating the terminal after a screen change. + + After the user types ``e`` to complete ``name``:: + + Before: >>> def greet(nam|): + ▲ + LineUpdate here: insert_char "e" + + After: >>> def greet(name|): + ▲ + + Only the changed cells are sent to the terminal; unchanged rows + are skipped entirely. + """ + + grow_lines: int + """Number of blank lines to append at the bottom to accommodate new content.""" + use_tall_mode: bool + """Use absolute cursor addressing via ``cup`` instead of relative moves. + Activated when content exceeds one screen height.""" + offset: int + """Vertical scroll offset: the buffer row displayed at the top of the terminal window.""" + reverse_scroll: int + """Number of lines to scroll backwards (content moves down).""" + forward_scroll: int + """Number of lines to scroll forwards (content moves up).""" + line_updates: tuple[LineUpdate, ...] + cleared_lines: tuple[int, ...] + """Row indices to erase (old content with no replacement).""" + rendered_screen: RenderedScreen + cursor: tuple[int, int] + + class UnixConsole(Console): + __buffer: list[_PendingWrite] + __gone_tall: bool + __move: _MoveFunc + __offset: int + def __init__( self, f_in: IO[bytes] | int = 0, @@ -159,9 +212,20 @@ def __init__( self.pollob.register(self.input_fd, select.POLLIN) self.terminfo = terminfo.TermInfo(term or None) self.term = term + self.is_apple_terminal = ( + platform.system() == "Darwin" + and os.getenv("TERM_PROGRAM") == "Apple_Terminal" + ) + + try: + self.__input_fd_set(tcgetattr(self.input_fd), ignore=frozenset()) + except _error as e: + raise RuntimeError(f"termios failure ({e.args[1]})") @overload - def _my_getstr(cap: str, optional: Literal[False] = False) -> bytes: ... + def _my_getstr( + cap: str, optional: Literal[False] = False + ) -> bytes: ... @overload def _my_getstr(cap: str, optional: bool) -> bytes | None: ... @@ -201,8 +265,10 @@ def _my_getstr(cap: str, optional: bool = False) -> bytes | None: self.__setup_movement() - self.event_queue = EventQueue(self.input_fd, self.encoding, self.terminfo) - self.cursor_visible = 1 + self.event_queue = EventQueue( + self.input_fd, self.encoding, self.terminfo + ) + self.cursor_visible = True signal.signal(signal.SIGCONT, self._sigcont_handler) @@ -213,7 +279,6 @@ def _sigcont_handler(self, signum, frame): def __read(self, n: int) -> bytes: return os.read(self.input_fd, n) - def change_encoding(self, encoding: str) -> None: """ Change the encoding used for I/O operations. @@ -223,33 +288,50 @@ def change_encoding(self, encoding: str) -> None: """ self.encoding = encoding - def refresh(self, screen, c_xy): + def refresh(self, rendered_screen: RenderedScreen) -> None: """ Refresh the console screen. Parameters: - - screen (list): List of strings representing the screen contents. - - c_xy (tuple): Cursor position (x, y) on the screen. + - rendered_screen: Structured rendered screen contents and cursor. """ + c_xy = rendered_screen.cursor + trace( + "unix.refresh start cursor={cursor} lines={lines} prev_lines={prev_lines} " + "offset={offset} posxy={posxy}", + cursor=c_xy, + lines=len(rendered_screen.composed_lines), + prev_lines=len(self._rendered_screen.composed_lines), + offset=self.__offset, + posxy=self.posxy, + ) + plan = self.__plan_refresh(rendered_screen, c_xy) + self.__apply_refresh_plan(plan) + + def __plan_refresh( + self, + rendered_screen: RenderedScreen, + c_xy: tuple[int, int], + ) -> UnixRefreshPlan: cx, cy = c_xy - if not self.__gone_tall: - while len(self.screen) < min(len(screen), self.height): - self.__hide_cursor() - self.__move(0, len(self.screen) - 1) - self.__write("\n") - self.posxy = 0, len(self.screen) - self.screen.append("") - else: - while len(self.screen) < len(screen): - self.screen.append("") - - if len(screen) > self.height: - self.__gone_tall = 1 - self.__move = self.__move_tall - - px, py = self.posxy - old_offset = offset = self.__offset height = self.height + old_offset = offset = self.__offset + prev_composed = self._rendered_screen.composed_lines + previous_lines = list(prev_composed) + next_lines = list(rendered_screen.composed_lines) + line_count = len(next_lines) + + grow_lines = 0 + if not self.__gone_tall: + grow_lines = max( + min(line_count, height) - len(prev_composed), + 0, + ) + previous_lines.extend([EMPTY_RENDER_LINE] * grow_lines) + elif len(previous_lines) < line_count: + previous_lines.extend([EMPTY_RENDER_LINE] * (line_count - len(previous_lines))) + + use_tall_mode = self.__gone_tall or line_count > height # we make sure the cursor is on the screen, and that we're # using all of the screen if we can @@ -257,56 +339,115 @@ def refresh(self, screen, c_xy): offset = cy elif cy >= offset + height: offset = cy - height + 1 - elif offset > 0 and len(screen) < offset + height: - offset = max(len(screen) - height, 0) - screen.append("") + elif offset > 0 and line_count < offset + height: + offset = max(line_count - height, 0) + next_lines.append(EMPTY_RENDER_LINE) - oldscr = self.screen[old_offset : old_offset + height] - newscr = screen[offset : offset + height] + oldscr = previous_lines[old_offset : old_offset + height] + newscr = next_lines[offset : offset + height] - # use hardware scrolling if we have it. + reverse_scroll = 0 + forward_scroll = 0 if old_offset > offset and self._ri: + reverse_scroll = old_offset - offset + for _ in range(reverse_scroll): + if oldscr: + oldscr.pop(-1) + oldscr.insert(0, EMPTY_RENDER_LINE) + elif old_offset < offset and self._ind: + forward_scroll = offset - old_offset + for _ in range(forward_scroll): + if oldscr: + oldscr.pop(0) + oldscr.append(EMPTY_RENDER_LINE) + + line_updates: list[LineUpdate] = [] + px, _ = self.posxy + for y, oldline, newline in zip(range(offset, offset + height), oldscr, newscr): + update = self.__plan_changed_line(y, oldline, newline, px) + if update is not None: + line_updates.append(update) + + cleared_lines = tuple(range(offset + len(newscr), offset + len(oldscr))) + console_rendered_screen = RenderedScreen(tuple(next_lines), c_xy) + trace( + "unix.refresh plan grow={grow} tall={tall} offset={offset} " + "reverse_scroll={reverse_scroll} forward_scroll={forward_scroll} " + "updates={updates} clears={clears}", + grow=grow_lines, + tall=use_tall_mode, + offset=offset, + reverse_scroll=reverse_scroll, + forward_scroll=forward_scroll, + updates=len(line_updates), + clears=len(cleared_lines), + ) + return UnixRefreshPlan( + grow_lines=grow_lines, + use_tall_mode=use_tall_mode, + offset=offset, + reverse_scroll=reverse_scroll, + forward_scroll=forward_scroll, + line_updates=tuple(line_updates), + cleared_lines=cleared_lines, + rendered_screen=console_rendered_screen, + cursor=(cx, cy), + ) + + def __apply_refresh_plan(self, plan: UnixRefreshPlan) -> None: + cx, cy = plan.cursor + trace( + "unix.refresh apply cursor={cursor} updates={updates} clears={clears}", + cursor=plan.cursor, + updates=len(plan.line_updates), + clears=len(plan.cleared_lines), + ) + visual_style = self.begin_redraw_visualization() + screen_line_count = len(self._rendered_screen.composed_lines) + + for _ in range(plan.grow_lines): + self.__hide_cursor() + if screen_line_count: + self.__move(0, screen_line_count - 1) + self.__write("\n") + self.posxy = 0, screen_line_count + screen_line_count += 1 + + if plan.use_tall_mode and not self.__gone_tall: + self.__gone_tall = True + self.__move = self.__move_tall + + old_offset = self.__offset + if plan.reverse_scroll: self.__hide_cursor() self.__write_code(self._cup, 0, 0) self.posxy = 0, old_offset - for i in range(old_offset - offset): + for _ in range(plan.reverse_scroll): self.__write_code(self._ri) - oldscr.pop(-1) - oldscr.insert(0, "") - elif old_offset < offset and self._ind: + elif plan.forward_scroll: self.__hide_cursor() self.__write_code(self._cup, self.height - 1, 0) self.posxy = 0, old_offset + self.height - 1 - for i in range(offset - old_offset): + for _ in range(plan.forward_scroll): self.__write_code(self._ind) - oldscr.pop(0) - oldscr.append("") - self.__offset = offset + self.__offset = plan.offset - for ( - y, - oldline, - newline, - ) in zip(range(offset, offset + height), oldscr, newscr): - if oldline != newline: - self.__write_changed_line(y, oldline, newline, px) + for update in plan.line_updates: + self.__apply_line_update(update, visual_style) - y = len(newscr) - while y < len(oldscr): + for y in plan.cleared_lines: self.__hide_cursor() self.__move(0, y) self.posxy = 0, y self.__write_code(self._el) - y += 1 self.__show_cursor() - - self.screen = screen.copy() self.move_cursor(cx, cy) self.flushoutput() + self.sync_rendered_screen(plan.rendered_screen, self.posxy) - def move_cursor(self, x, y): + def move_cursor(self, x: int, y: int) -> None: """ Move the cursor to the specified position on the screen. @@ -315,16 +456,27 @@ def move_cursor(self, x, y): - y (int): Y coordinate. """ if y < self.__offset or y >= self.__offset + self.height: - self.event_queue.insert(Event("scroll", None)) + trace( + "unix.move_cursor offscreen x={x} y={y} offset={offset} height={height}", + x=x, + y=y, + offset=self.__offset, + height=self.height, + ) + self.event_queue.insert(Event("scroll", "")) else: + trace("unix.move_cursor x={x} y={y}", x=x, y=y) self.__move(x, y) self.posxy = x, y self.flushoutput() - def prepare(self): + def prepare(self) -> None: """ Prepare the console for input/output operations. """ + trace("unix.prepare") + self.__buffer = [] + self.__svtermstate = tcgetattr(self.input_fd) raw = self.__svtermstate.copy() raw.iflag &= ~(termios.INPCK | termios.ISTRIP | termios.IXON) @@ -334,23 +486,22 @@ def prepare(self): raw.iflag |= termios.BRKINT raw.lflag &= ~(termios.ICANON | termios.ECHO | termios.IEXTEN) raw.lflag |= termios.ISIG - raw.cc[termios.VMIN] = 1 - raw.cc[termios.VTIME] = 0 - tcsetattr(self.input_fd, termios.TCSADRAIN, raw) + raw.cc[termios.VMIN] = b"\x01" + raw.cc[termios.VTIME] = b"\x00" + self.__input_fd_set(raw) - # In macOS terminal we need to deactivate line wrap via ANSI escape code - if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + # Apple Terminal will re-wrap lines for us unless we preempt the + # damage. + if self.is_apple_terminal: os.write(self.output_fd, b"\033[?7l") - self.screen = [] self.height, self.width = self.getheightwidth() - self.__buffer = [] - self.posxy = 0, 0 - self.__gone_tall = 0 + self.__gone_tall = False self.__move = self.__move_short self.__offset = 0 + self.sync_rendered_screen(RenderedScreen.empty(), self.posxy) self.__maybe_write_code(self._smkx) @@ -361,20 +512,26 @@ def prepare(self): self.__enable_bracketed_paste() - def restore(self): + def restore(self) -> None: """ Restore the console to the default state """ + trace("unix.restore") self.__disable_bracketed_paste() self.__maybe_write_code(self._rmkx) self.flushoutput() - tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate) + self.__input_fd_set(self.__svtermstate) - if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + if self.is_apple_terminal: os.write(self.output_fd, b"\033[?7h") if hasattr(self, "old_sigwinch"): - signal.signal(signal.SIGWINCH, self.old_sigwinch) + try: + signal.signal(signal.SIGWINCH, self.old_sigwinch) + except ValueError as e: + import threading + if threading.current_thread() is threading.main_thread(): + raise e del self.old_sigwinch def push_char(self, char: int | bytes) -> None: @@ -407,6 +564,8 @@ def get_event(self, block: bool = True) -> Event | None: return self.event_queue.get() else: continue + elif err.errno == errno.EIO: + raise SystemExit(errno.EIO) else: raise else: @@ -422,7 +581,7 @@ def wait(self, timeout: float | None = None) -> bool: or bool(self.pollob.poll(timeout)) ) - def set_cursor_vis(self, visible): + def set_cursor_vis(self, visible: bool) -> None: """ Set the visibility of the cursor. @@ -490,8 +649,9 @@ def finish(self): """ Finish console operations and flush the output buffer. """ - y = len(self.screen) - 1 - while y >= 0 and not self.screen[y]: + rendered_lines = self._rendered_screen.composed_lines + y = len(rendered_lines) - 1 + while y >= 0 and not rendered_lines[y].text: y -= 1 self.__move(0, min(y, self.height + self.__offset - 1)) self.__write("\n\r") @@ -518,7 +678,7 @@ def getpending(self): while not self.event_queue.empty(): e2 = self.event_queue.get() e.data += e2.data - e.raw += e.raw + e.raw += e2.raw amount = struct.unpack("i", ioctl(self.input_fd, FIONREAD, b"\0\0\0\0"))[0] trace("getpending({a})", a=amount) @@ -542,7 +702,7 @@ def getpending(self): while not self.event_queue.empty(): e2 = self.event_queue.get() e.data += e2.data - e.raw += e.raw + e.raw += e2.raw amount = 10000 raw = self.__read(amount) @@ -555,11 +715,12 @@ def clear(self): """ Clear the console screen. """ + trace("unix.clear") self.__write_code(self._clear) - self.__gone_tall = 1 + self.__gone_tall = True self.__move = self.__move_tall self.posxy = 0, 0 - self.screen = [] + self.sync_rendered_screen(RenderedScreen.empty(), self.posxy) @property def input_hook(self): @@ -610,98 +771,178 @@ def __setup_movement(self): self.__move = self.__move_short - def __write_changed_line(self, y, oldline, newline, px_coord): - # this is frustrating; there's no reason to test (say) - # self.dch1 inside the loop -- but alternative ways of - # structuring this function are equally painful (I'm trying to - # avoid writing code generators these days...) - minlen = min(wlen(oldline), wlen(newline)) - x_pos = 0 - x_coord = 0 + @staticmethod + def __cell_index_from_x(line: RenderLine, x_coord: int) -> int: + width = 0 + index = 0 + while index < len(line.cells) and width < x_coord: + width += line.cells[index].width + index += 1 + return index - px_pos = 0 - j = 0 - for c in oldline: - if j >= px_coord: - break - j += wlen(c) - px_pos += 1 + def __plan_changed_line( + self, + y: int, + oldline: RenderLine, + newline: RenderLine, + px_coord: int, + ) -> LineUpdate | None: + # NOTE: The shared replace_char / replace_span / rewrite_suffix logic + # is duplicated in WindowsConsole.__plan_changed_line. Keep changes to + # these common cases synchronised between the two files. Yes, this is + # duplicated on purpose; the two backends agree just enough to make a + # shared helper a trap. Unix-only cases (insert_char, delete_then_insert) + # rely on terminal capabilities (ich1/dch1) that are unavailable on + # Windows. + diff = diff_render_lines(oldline, newline) + if diff is None: + return None - # reuse the oldline as much as possible, but stop as soon as we - # encounter an ESCAPE, because it might be the start of an escape - # sequence - while ( - x_coord < minlen - and oldline[x_pos] == newline[x_pos] - and newline[x_pos] != "\x1b" + start_cell = diff.start_cell + start_x = diff.start_x + + if ( + self.ich1 + and not diff.old_cells + and (visible_new_cells := tuple( + cell for cell in diff.new_cells if cell.width + )) + and len(visible_new_cells) == 1 + and all(cell.width == 0 for cell in diff.new_cells[1:]) + and oldline.cells[start_cell:] == newline.cells[start_cell + 1 :] ): - x_coord += wlen(newline[x_pos]) - x_pos += 1 - - # if we need to insert a single character right after the first detected change - if oldline[x_pos:] == newline[x_pos + 1 :] and self.ich1: + px_cell = self.__cell_index_from_x(oldline, px_coord) if ( y == self.posxy[1] - and x_coord > self.posxy[0] - and oldline[px_pos:x_pos] == newline[px_pos + 1 : x_pos + 1] + and start_x > self.posxy[0] + and oldline.cells[px_cell:start_cell] + == newline.cells[px_cell + 1 : start_cell + 1] ): - x_pos = px_pos - x_coord = px_coord - character_width = wlen(newline[x_pos]) - self.__move(x_coord, y) - self.__write_code(self.ich1) - self.__write(newline[x_pos]) - self.posxy = x_coord + character_width, y + start_cell = px_cell + start_x = px_coord + planned_cells = diff.new_cells + changed_cell = visible_new_cells[0] + return LineUpdate( + kind="insert_char", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=planned_cells, + char_width=changed_cell.width, + reset_to_margin=requires_cursor_resync(planned_cells), + ) - # if it's a single character change in the middle of the line - elif ( - x_coord < minlen - and oldline[x_pos + 1 :] == newline[x_pos + 1 :] - and wlen(oldline[x_pos]) == wlen(newline[x_pos]) + if ( + len(diff.old_cells) == 1 + and len(diff.new_cells) == 1 + and diff.old_cells[0].width == diff.new_cells[0].width ): - character_width = wlen(newline[x_pos]) - self.__move(x_coord, y) - self.__write(newline[x_pos]) - self.posxy = x_coord + character_width, y + planned_cells = diff.new_cells + changed_cell = planned_cells[0] + return LineUpdate( + kind="replace_char", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=planned_cells, + char_width=changed_cell.width, + reset_to_margin=requires_cursor_resync(planned_cells), + ) - # if this is the last character to fit in the line and we edit in the middle of the line - elif ( + if diff.old_changed_width == diff.new_changed_width: + planned_cells = diff.new_cells + return LineUpdate( + kind="replace_span", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=planned_cells, + char_width=diff.new_changed_width, + reset_to_margin=requires_cursor_resync(planned_cells), + ) + + if ( self.dch1 and self.ich1 - and wlen(newline) == self.width - and x_coord < wlen(newline) - 2 - and newline[x_pos + 1 : -1] == oldline[x_pos:-2] + and newline.width == self.width + and start_x < newline.width - 2 + and newline.cells[start_cell + 1 : -1] == oldline.cells[start_cell:-2] ): - self.__hide_cursor() - self.__move(self.width - 2, y) - self.posxy = self.width - 2, y - self.__write_code(self.dch1) + planned_cells = (newline.cells[start_cell],) + changed_cell = planned_cells[0] + return LineUpdate( + kind="delete_then_insert", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=planned_cells, + char_width=changed_cell.width, + reset_to_margin=requires_cursor_resync(planned_cells), + ) - character_width = wlen(newline[x_pos]) - self.__move(x_coord, y) + suffix_cells = newline.cells[start_cell:] + return LineUpdate( + kind="rewrite_suffix", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=suffix_cells, + char_width=sum(cell.width for cell in suffix_cells), + clear_eol=oldline.width > newline.width, + reset_to_margin=requires_cursor_resync(suffix_cells), + ) + + def __apply_line_update( + self, + update: LineUpdate, + visual_style: str | None = None, + ) -> None: + text = render_cells(update.cells, visual_style) if visual_style else update.text + trace( + "unix.refresh update kind={kind} y={y} x={x} text={text} " + "clear_eol={clear_eol} reset_to_margin={reset}", + kind=update.kind, + y=update.y, + x=update.start_x, + text=trace_text(text), + clear_eol=update.clear_eol, + reset=update.reset_to_margin, + ) + if update.kind == "insert_char": + self.__move(update.start_x, update.y) self.__write_code(self.ich1) - self.__write(newline[x_pos]) - self.posxy = character_width + 1, y - + self.__write(text) + self.posxy = update.start_x + update.char_width, update.y + elif update.kind in {"replace_char", "replace_span"}: + self.__move(update.start_x, update.y) + self.__write(text) + self.posxy = update.start_x + update.char_width, update.y + elif update.kind == "delete_then_insert": + self.__hide_cursor() + self.__move(self.width - 2, update.y) + self.posxy = self.width - 2, update.y + self.__write_code(self.dch1) + self.__move(update.start_x, update.y) + self.__write_code(self.ich1) + self.__write(text) + self.posxy = update.start_x + update.char_width, update.y else: self.__hide_cursor() - self.__move(x_coord, y) - if wlen(oldline) > wlen(newline): + self.__move(update.start_x, update.y) + if update.clear_eol: self.__write_code(self._el) - self.__write(newline[x_pos:]) - self.posxy = wlen(newline), y + self.__write(text) + self.posxy = update.start_x + update.char_width, update.y - if "\x1b" in newline: - # ANSI escape characters are present, so we can't assume - # anything about the position of the cursor. Moving the cursor - # to the left margin should work to get to a known position. - self.move_cursor(0, y) + if update.reset_to_margin: + # Non-SGR terminal controls can affect the cursor position. + self.move_cursor(0, update.y) def __write(self, text): - self.__buffer.append((text, 0)) + self.__buffer.append((text, False)) def __write_code(self, fmt, *args): - self.__buffer.append((terminfo.tparm(fmt, *args), 1)) + self.__buffer.append((terminfo.tparm(fmt, *args), True)) def __maybe_write_code(self, fmt, *args): if fmt: @@ -752,30 +993,38 @@ def __move_tall(self, x, y): self.__write_code(self._cup, y - self.__offset, x) def __sigwinch(self, signum, frame): - self.height, self.width = self.getheightwidth() - self.event_queue.insert(Event("resize", None)) + self.event_queue.insert(Event("resize", "")) def __hide_cursor(self): if self.cursor_visible: self.__maybe_write_code(self._civis) - self.cursor_visible = 0 + self.cursor_visible = False def __show_cursor(self): if not self.cursor_visible: self.__maybe_write_code(self._cnorm) - self.cursor_visible = 1 + self.cursor_visible = True def repaint(self): + composed = self._rendered_screen.composed_lines + trace( + "unix.repaint gone_tall={gone_tall} screen_lines={lines} offset={offset}", + gone_tall=self.__gone_tall, + lines=len(composed), + offset=self.__offset, + ) if not self.__gone_tall: self.posxy = 0, self.posxy[1] self.__write("\r") - ns = len(self.screen) * ["\000" * self.width] - self.screen = ns + ns = len(composed) * ["\000" * self.width] else: self.posxy = 0, self.__offset self.__move(0, self.__offset) ns = self.height * ["\000" * self.width] - self.screen = ns + self.sync_rendered_screen( + RenderedScreen.from_screen_lines(ns, self.posxy), + self.posxy, + ) def __tputs(self, fmt, prog=delayprog): """A Python implementation of the curses tputs function; the @@ -785,7 +1034,7 @@ def __tputs(self, fmt, prog=delayprog): will never do anyone any good.""" # using .get() means that things will blow up # only if the bps is actually needed (which I'm - # betting is pretty unlkely) + # betting is pretty unlikely) bps = ratedict.get(self.__svtermstate.ospeed) while True: m = prog.search(fmt) @@ -803,3 +1052,17 @@ def __tputs(self, fmt, prog=delayprog): os.write(self.output_fd, self._pad * nchars) else: time.sleep(float(delay) / 1000.0) + + def __input_fd_set( + self, + state: TermState, + ignore: AbstractSet[int] = _error_codes_to_ignore, + ) -> bool: + try: + tcsetattr(self.input_fd, termios.TCSADRAIN, state) + except termios.error as te: + if te.args[0] not in ignore: + raise + return False + else: + return True diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index c5d006afa77..b50426c31ea 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -9,6 +9,7 @@ import _colorize from collections import deque +from dataclasses import dataclass from io import StringIO from tokenize import TokenInfo as TI from typing import Iterable, Iterator, Match, NamedTuple, Self @@ -16,11 +17,13 @@ from .types import CharBuffer, CharWidths from .trace import trace + ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]") ZERO_WIDTH_BRACKET = re.compile(r"\x01.*?\x02") ZERO_WIDTH_TRANS = str.maketrans({"\x01": "", "\x02": ""}) -IDENTIFIERS_AFTER = {"def", "class"} -BUILTINS = {str(name) for name in dir(builtins) if not name.startswith('_')} +IDENTIFIERS_AFTER = frozenset({"def", "class"}) +KEYWORD_CONSTANTS = frozenset({"True", "False", "None"}) +BUILTINS = frozenset({str(name) for name in dir(builtins) if not name.startswith('_')}) def THEME(**kwargs): @@ -58,10 +61,31 @@ class ColorSpan(NamedTuple): tag: str +class StyledChar(NamedTuple): + text: str + width: int + tag: str | None = None + + +def _ascii_control_repr(c: str) -> str | None: + code = ord(c) + if code < 32: + return "^" + chr(code + 64) + if code == 127: + return "^?" + return None + + @functools.cache def str_width(c: str) -> int: if ord(c) < 128: return 1 + # gh-139246 for zero-width joiner and combining characters + if unicodedata.combining(c): + return 0 + category = unicodedata.category(c) + if category == "Cf" and c != "\u00ad": + return 0 w = unicodedata.east_asian_width(c) if w in ("N", "Na", "H", "A"): return 1 @@ -197,8 +221,11 @@ def gen_colors_from_token_stream( span = Span.from_token(token, line_lengths) yield ColorSpan(span, "definition") elif keyword.iskeyword(token.string): + span_cls = "keyword" + if token.string in KEYWORD_CONSTANTS: + span_cls = "keyword_constant" span = Span.from_token(token, line_lengths) - yield ColorSpan(span, "keyword") + yield ColorSpan(span, span_cls) if token.string in IDENTIFIERS_AFTER: is_def_name = True elif ( @@ -216,8 +243,8 @@ def gen_colors_from_token_stream( yield ColorSpan(span, "builtin") -keyword_first_sets_match = {"False", "None", "True", "await", "lambda", "not"} -keyword_first_sets_case = {"False", "None", "True"} +keyword_first_sets_match = frozenset({"False", "None", "True", "await", "lambda", "not"}) +keyword_first_sets_case = frozenset({"False", "None", "True"}) def is_soft_keyword_used(*tokens: TI | None) -> bool: @@ -260,10 +287,77 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: return True case (TI(string="case"), TI(string="_"), TI(string=":")): return True + case ( + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), + TI(string="type"), + TI(T.NAME, string=s) + ): + return not keyword.iskeyword(s) + case ( + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":" | ";"), + TI(string="lazy"), + TI(string="import") | TI(string="from") + ): + return True case _: return False +def iter_display_chars( + buffer: str, + colors: list[ColorSpan] | None = None, + start_index: int = 0, +) -> Iterator[StyledChar]: + """Yield visible display characters with widths and semantic color tags. + + Note: ``colors`` is consumed in place as spans are processed -- callers + that split a buffer across multiple calls rely on this mutation to track + which spans have already been handled. + """ + + if not buffer: + return + + color_idx = 0 + if colors: + while color_idx < len(colors) and colors[color_idx].span.end < start_index: + color_idx += 1 + + active_tag = None + if colors and color_idx < len(colors) and colors[color_idx].span.start < start_index: + active_tag = colors[color_idx].tag + + for i, c in enumerate(buffer, start_index): + if colors and color_idx < len(colors) and colors[color_idx].span.start == i: + active_tag = colors[color_idx].tag + + if control := _ascii_control_repr(c): + text = control + width = len(control) + elif ord(c) < 128: + text = c + width = 1 + elif unicodedata.category(c).startswith("C"): + text = r"\u%04x" % ord(c) + width = len(text) + else: + text = c + width = str_width(c) + + yield StyledChar(text, width, active_tag) + + if colors and color_idx < len(colors) and colors[color_idx].span.end == i: + color_idx += 1 + active_tag = None + # Check if the next span starts at the same position + if color_idx < len(colors) and colors[color_idx].span.start == i: + active_tag = colors[color_idx].tag + + # Remove consumed spans so callers see the mutation + if color_idx > 0 and colors: + del colors[:color_idx] + + def disp_str( buffer: str, colors: list[ColorSpan] | None = None, @@ -299,53 +393,18 @@ def disp_str( (['\x1b[1;34mw', 'h', 'i', 'l', 'e\x1b[0m', ' ', '1', ':'], [1, 1, 1, 1, 1, 1, 1, 1]) """ + styled_chars = list(iter_display_chars(buffer, colors, start_index)) chars: CharBuffer = [] char_widths: CharWidths = [] - - if not buffer: - return chars, char_widths - - while colors and colors[0].span.end < start_index: - # move past irrelevant spans - colors.pop(0) - theme = THEME(force_color=force_color) - pre_color = "" - post_color = "" - if colors and colors[0].span.start < start_index: - # looks like we're continuing a previous color (e.g. a multiline str) - pre_color = theme[colors[0].tag] - for i, c in enumerate(buffer, start_index): - if colors and colors[0].span.start == i: # new color starts now - pre_color = theme[colors[0].tag] - - if c == "\x1a": # CTRL-Z on Windows - chars.append(c) - char_widths.append(2) - elif ord(c) < 128: - chars.append(c) - char_widths.append(1) - elif unicodedata.category(c).startswith("C"): - c = r"\u%04x" % ord(c) - chars.append(c) - char_widths.append(len(c)) - else: - chars.append(c) - char_widths.append(str_width(c)) - - if colors and colors[0].span.end == i: # current color ends now - post_color = theme.reset - colors.pop(0) - - chars[-1] = pre_color + chars[-1] + post_color - pre_color = "" - post_color = "" - - if colors and colors[0].span.start < i and colors[0].span.end > i: - # even though the current color should be continued, reset it for now. - # the next call to `disp_str()` will revive it. - chars[-1] += theme.reset + for index, styled_char in enumerate(styled_chars): + previous_tag = styled_chars[index - 1].tag if index else None + next_tag = styled_chars[index + 1].tag if index + 1 < len(styled_chars) else None + prefix = theme[styled_char.tag] if styled_char.tag and styled_char.tag != previous_tag else "" + suffix = theme.reset if styled_char.tag and styled_char.tag != next_tag else "" + chars.append(prefix + styled_char.text + suffix) + char_widths.append(styled_char.width) return chars, char_widths @@ -363,13 +422,35 @@ def prev_next_window[T]( """ iterator = iter(iterable) - window = deque((None, next(iterator)), maxlen=3) + try: + first = next(iterator) + except StopIteration: + return + window = deque((None, first), maxlen=3) try: for x in iterator: window.append(x) yield tuple(window) - except Exception: - raise finally: window.append(None) yield tuple(window) + + +@dataclass(frozen=True, slots=True) +class StyleRef: + tag: str | None = None # From THEME().syntax, e.g. "keyword", "builtin" + sgr: str = "" + + @classmethod + def from_tag(cls, tag: str, sgr: str = "") -> Self: + return cls(tag=tag, sgr=sgr) + + @classmethod + def from_sgr(cls, sgr: str) -> Self: + if not sgr: + return cls() + return cls(sgr=sgr) + + @property + def is_plain(self) -> bool: + return self.tag is None and not self.sgr diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index c56dcd6d7dd..c1f9a19545d 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -25,6 +25,7 @@ import ctypes import types +from dataclasses import dataclass from ctypes.wintypes import ( _COORD, WORD, @@ -37,20 +38,26 @@ SHORT, ) from ctypes import Structure, POINTER, Union +from typing import TYPE_CHECKING from .console import Event, Console -from .trace import trace -from .utils import wlen +from .render import ( + EMPTY_RENDER_LINE, + LineUpdate, + RenderLine, + RenderedScreen, + requires_cursor_resync, + diff_render_lines, + render_cells, +) +from .trace import trace, trace_text from .windows_eventqueue import EventQueue try: - from ctypes import get_last_error, GetLastError, WinDLL, windll, WinError # type: ignore[attr-defined] + from ctypes import get_last_error, WinDLL, windll, WinError # type: ignore[attr-defined] except: # Keep MyPy happy off Windows from ctypes import CDLL as WinDLL, cdll as windll - def GetLastError() -> int: - return 42 - def get_last_error() -> int: return 42 @@ -66,8 +73,6 @@ def __init__(self, err: int | None, descr: str | None = None) -> None: except ImportError: nt = None -TYPE_CHECKING = False - if TYPE_CHECKING: from typing import IO @@ -126,6 +131,17 @@ def __init__(self, err: int | None, descr: str | None = None) -> None: class _error(Exception): pass + +@dataclass(frozen=True, slots=True) +class WindowsRefreshPlan: + grow_lines: int + offset: int + scroll_lines: int + line_updates: tuple[LineUpdate, ...] + cleared_lines: tuple[int, ...] + rendered_screen: RenderedScreen + cursor: tuple[int, int] + def _supports_vt(): try: return nt._supports_virtual_terminal() @@ -149,18 +165,19 @@ def __init__( # Save original console modes so we can recover on cleanup. original_input_mode = DWORD() - GetConsoleMode(InHandle, original_input_mode) + if not GetConsoleMode(InHandle, original_input_mode): + raise WinError(get_last_error()) trace(f'saved original input mode 0x{original_input_mode.value:x}') self.__original_input_mode = original_input_mode.value - SetConsoleMode( + if not SetConsoleMode( OutHandle, ENABLE_WRAP_AT_EOL_OUTPUT | ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING, - ) + ): + raise WinError(get_last_error()) - self.screen: list[str] = [] self.width = 80 self.height = 25 self.__offset = 0 @@ -171,73 +188,124 @@ def __init__( # Console I/O is redirected, fallback... self.out = None - def refresh(self, screen: list[str], c_xy: tuple[int, int]) -> None: + def refresh(self, rendered_screen: RenderedScreen) -> None: """ Refresh the console screen. Parameters: - - screen (list): List of strings representing the screen contents. - - c_xy (tuple): Cursor position (x, y) on the screen. + - rendered_screen: Structured rendered screen contents and cursor. """ + c_xy = rendered_screen.cursor + trace( + "windows.refresh start cursor={cursor} lines={lines} prev_lines={prev_lines} " + "offset={offset} posxy={posxy}", + cursor=c_xy, + lines=len(rendered_screen.composed_lines), + prev_lines=len(self._rendered_screen.composed_lines), + offset=self.__offset, + posxy=self.posxy, + ) + plan = self.__plan_refresh(rendered_screen, c_xy) + self.__apply_refresh_plan(plan) + + def __plan_refresh( + self, + rendered_screen: RenderedScreen, + c_xy: tuple[int, int], + ) -> WindowsRefreshPlan: cx, cy = c_xy - - while len(self.screen) < min(len(screen), self.height): - self._hide_cursor() - self._move_relative(0, len(self.screen) - 1) - self.__write("\n") - self.posxy = 0, len(self.screen) - self.screen.append("") - - px, py = self.posxy - old_offset = offset = self.__offset height = self.height + old_offset = offset = self.__offset + prev_composed = self._rendered_screen.composed_lines + previous_lines = list(prev_composed) + next_lines = list(rendered_screen.composed_lines) + line_count = len(next_lines) - # we make sure the cursor is on the screen, and that we're - # using all of the screen if we can + grow_lines = max( + min(line_count, height) - len(prev_composed), + 0, + ) + previous_lines.extend([EMPTY_RENDER_LINE] * grow_lines) + + scroll_lines = 0 if cy < offset: offset = cy elif cy >= offset + height: offset = cy - height + 1 scroll_lines = offset - old_offset + previous_lines.extend([EMPTY_RENDER_LINE] * scroll_lines) + elif offset > 0 and line_count < offset + height: + offset = max(line_count - height, 0) + next_lines.append(EMPTY_RENDER_LINE) - # Scrolling the buffer as the current input is greater than the visible - # portion of the window. We need to scroll the visible portion and the - # entire history - self._scroll(scroll_lines, self._getscrollbacksize()) - self.posxy = self.posxy[0], self.posxy[1] + scroll_lines - self.__offset += scroll_lines + oldscr = previous_lines[old_offset : old_offset + height] + newscr = next_lines[offset : offset + height] - for i in range(scroll_lines): - self.screen.append("") - elif offset > 0 and len(screen) < offset + height: - offset = max(len(screen) - height, 0) - screen.append("") + line_updates: list[LineUpdate] = [] + px, _ = self.posxy + for y, oldline, newline in zip(range(offset, offset + height), oldscr, newscr): + update = self.__plan_changed_line(y, oldline, newline, px) + if update is not None: + line_updates.append(update) - oldscr = self.screen[old_offset : old_offset + height] - newscr = screen[offset : offset + height] + cleared_lines = tuple(range(offset + len(newscr), offset + len(oldscr))) + console_rendered_screen = RenderedScreen(tuple(next_lines), c_xy) + trace( + "windows.refresh plan grow={grow} offset={offset} scroll_lines={scroll_lines} " + "updates={updates} clears={clears}", + grow=grow_lines, + offset=offset, + scroll_lines=scroll_lines, + updates=len(line_updates), + clears=len(cleared_lines), + ) + return WindowsRefreshPlan( + grow_lines=grow_lines, + offset=offset, + scroll_lines=scroll_lines, + line_updates=tuple(line_updates), + cleared_lines=cleared_lines, + rendered_screen=console_rendered_screen, + cursor=(cx, cy), + ) - self.__offset = offset + def __apply_refresh_plan(self, plan: WindowsRefreshPlan) -> None: + cx, cy = plan.cursor + trace( + "windows.refresh apply cursor={cursor} updates={updates} clears={clears}", + cursor=plan.cursor, + updates=len(plan.line_updates), + clears=len(plan.cleared_lines), + ) + visual_style = self.begin_redraw_visualization() + screen_line_count = len(self._rendered_screen.composed_lines) + + for _ in range(plan.grow_lines): + self._hide_cursor() + if screen_line_count: + self._move_relative(0, screen_line_count - 1) + self.__write("\n") + self.posxy = 0, screen_line_count + screen_line_count += 1 + + if plan.scroll_lines: + self._scroll(plan.scroll_lines, self._getscrollbacksize()) + self.posxy = self.posxy[0], self.posxy[1] + plan.scroll_lines + + self.__offset = plan.offset self._hide_cursor() - for ( - y, - oldline, - newline, - ) in zip(range(offset, offset + height), oldscr, newscr): - if oldline != newline: - self.__write_changed_line(y, oldline, newline, px) + for update in plan.line_updates: + self.__apply_line_update(update, visual_style) - y = len(newscr) - while y < len(oldscr): + for y in plan.cleared_lines: self._move_relative(0, y) self.posxy = 0, y self._erase_to_end() - y += 1 self._show_cursor() - - self.screen = screen self.move_cursor(cx, cy) + self.sync_rendered_screen(plan.rendered_screen, self.posxy) @property def input_hook(self): @@ -246,54 +314,98 @@ def input_hook(self): if nt is not None and nt._is_inputhook_installed(): return nt._inputhook - def __write_changed_line( - self, y: int, oldline: str, newline: str, px_coord: int - ) -> None: - # this is frustrating; there's no reason to test (say) - # self.dch1 inside the loop -- but alternative ways of - # structuring this function are equally painful (I'm trying to - # avoid writing code generators these days...) - minlen = min(wlen(oldline), wlen(newline)) - x_pos = 0 - x_coord = 0 + def __plan_changed_line( # keep in sync with UnixConsole.__plan_changed_line + self, + y: int, + oldline: RenderLine, + newline: RenderLine, + px_coord: int, + ) -> LineUpdate | None: + diff = diff_render_lines(oldline, newline) + if diff is None: + return None - px_pos = 0 - j = 0 - for c in oldline: - if j >= px_coord: - break - j += wlen(c) - px_pos += 1 - - # reuse the oldline as much as possible, but stop as soon as we - # encounter an ESCAPE, because it might be the start of an escape - # sequence - while ( - x_coord < minlen - and oldline[x_pos] == newline[x_pos] - and newline[x_pos] != "\x1b" + start_cell = diff.start_cell + start_x = diff.start_x + if ( + len(diff.old_cells) == 1 + and len(diff.new_cells) == 1 + and diff.old_cells[0].width == diff.new_cells[0].width ): - x_coord += wlen(newline[x_pos]) - x_pos += 1 + changed_cell = diff.new_cells[0] + # Ctrl-Z (SUB) can reach here via RenderLine.from_rendered_text() + # for prompt/message lines, which bypasses iter_display_chars(). + # On Windows, raw \x1a causes console cursor anomalies, so we + # force a cursor resync when it appears. + return LineUpdate( + kind="replace_char", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=diff.new_cells, + char_width=changed_cell.width, + reset_to_margin=( + requires_cursor_resync(diff.new_cells) + or "\x1a" in changed_cell.text + ), + ) - self._hide_cursor() - self._move_relative(x_coord, y) - if wlen(oldline) > wlen(newline): + if diff.old_changed_width == diff.new_changed_width: + return LineUpdate( + kind="replace_span", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=diff.new_cells, + char_width=diff.new_changed_width, + reset_to_margin=( + requires_cursor_resync(diff.new_cells) + or any("\x1a" in cell.text for cell in diff.new_cells) + ), + ) + + suffix_cells = newline.cells[start_cell:] + return LineUpdate( + kind="rewrite_suffix", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=suffix_cells, + char_width=sum(cell.width for cell in suffix_cells), + clear_eol=oldline.width > newline.width, + reset_to_margin=( + requires_cursor_resync(suffix_cells) + or any("\x1a" in cell.text for cell in suffix_cells) + ), + ) + + def __apply_line_update( + self, + update: LineUpdate, + visual_style: str | None = None, + ) -> None: + text = render_cells(update.cells, visual_style) if visual_style else update.text + trace( + "windows.refresh update kind={kind} y={y} x={x} text={text} " + "clear_eol={clear_eol} reset_to_margin={reset}", + kind=update.kind, + y=update.y, + x=update.start_x, + text=trace_text(text), + clear_eol=update.clear_eol, + reset=update.reset_to_margin, + ) + original_y = self.posxy[1] + self._move_relative(update.start_x, update.y) + if update.clear_eol: self._erase_to_end() - self.__write(newline[x_pos:]) - if wlen(newline) == self.width: - # If we wrapped we want to start at the next line - self._move_relative(0, y + 1) - self.posxy = 0, y + 1 - else: - self.posxy = wlen(newline), y + self.__write(text) + self.posxy = min(update.start_x + update.char_width, self.width - 1), update.y - if "\x1b" in newline or y != self.posxy[1] or '\x1a' in newline: - # ANSI escape characters are present, so we can't assume - # anything about the position of the cursor. Moving the cursor - # to the left margin should work to get to a known position. - self.move_cursor(0, y) + if update.reset_to_margin or update.y != original_y: + # Non-SGR terminal controls or vertical movement require a cursor sync. + self.move_cursor(0, update.y) def _scroll( self, top: int, bottom: int, left: int | None = None, right: int | None = None @@ -312,7 +424,7 @@ def _scroll( if not ScrollConsoleScreenBuffer( OutHandle, scroll_rect, None, destination_origin, fill_info ): - raise WinError(GetLastError()) + raise WinError(get_last_error()) def _hide_cursor(self): self.__write("\x1b[?25l") @@ -334,7 +446,7 @@ def _disable_bracketed_paste(self) -> None: def __write(self, text: str) -> None: if "\x1a" in text: - text = ''.join(["^Z" if x == '\x1a' else x for x in text]) + text = text.replace("\x1a", "^Z") if self.out is not None: self.out.write(text.encode(self.encoding, "replace")) @@ -346,30 +458,32 @@ def __write(self, text: str) -> None: def screen_xy(self) -> tuple[int, int]: info = CONSOLE_SCREEN_BUFFER_INFO() if not GetConsoleScreenBufferInfo(OutHandle, info): - raise WinError(GetLastError()) + raise WinError(get_last_error()) return info.dwCursorPosition.X, info.dwCursorPosition.Y def _erase_to_end(self) -> None: self.__write(ERASE_IN_LINE) def prepare(self) -> None: - trace("prepare") - self.screen = [] + trace("windows.prepare") self.height, self.width = self.getheightwidth() self.posxy = 0, 0 - self.__gone_tall = 0 self.__offset = 0 + self.sync_rendered_screen(RenderedScreen.empty(), self.posxy) if self.__vt_support: - SetConsoleMode(InHandle, self.__original_input_mode | ENABLE_VIRTUAL_TERMINAL_INPUT) + if not SetConsoleMode(InHandle, self.__original_input_mode | ENABLE_VIRTUAL_TERMINAL_INPUT): + raise WinError(get_last_error()) self._enable_bracketed_paste() def restore(self) -> None: + trace("windows.restore") if self.__vt_support: # Recover to original mode before running REPL self._disable_bracketed_paste() - SetConsoleMode(InHandle, self.__original_input_mode) + if not SetConsoleMode(InHandle, self.__original_input_mode): + raise WinError(get_last_error()) def _move_relative(self, x: int, y: int) -> None: """Moves relative to the current posxy""" @@ -390,8 +504,16 @@ def move_cursor(self, x: int, y: int) -> None: raise ValueError(f"Bad cursor position {x}, {y}") if y < self.__offset or y >= self.__offset + self.height: + trace( + "windows.move_cursor offscreen x={x} y={y} offset={offset} height={height}", + x=x, + y=y, + offset=self.__offset, + height=self.height, + ) self.event_queue.insert(Event("scroll", "")) else: + trace("windows.move_cursor x={x} y={y}", x=x, y=y) self._move_relative(x, y) self.posxy = x, y @@ -406,7 +528,7 @@ def getheightwidth(self) -> tuple[int, int]: and width of the terminal window in characters.""" info = CONSOLE_SCREEN_BUFFER_INFO() if not GetConsoleScreenBufferInfo(OutHandle, info): - raise WinError(GetLastError()) + raise WinError(get_last_error()) return ( info.srWindow.Bottom - info.srWindow.Top + 1, info.srWindow.Right - info.srWindow.Left + 1, @@ -415,7 +537,7 @@ def getheightwidth(self) -> tuple[int, int]: def _getscrollbacksize(self) -> int: info = CONSOLE_SCREEN_BUFFER_INFO() if not GetConsoleScreenBufferInfo(OutHandle, info): - raise WinError(GetLastError()) + raise WinError(get_last_error()) return info.srWindow.Bottom # type: ignore[no-any-return] @@ -423,7 +545,7 @@ def _read_input(self) -> INPUT_RECORD | None: rec = INPUT_RECORD() read = DWORD() if not ReadConsoleInput(InHandle, rec, 1, read): - raise WinError(GetLastError()) + raise WinError(get_last_error()) return rec @@ -433,7 +555,7 @@ def _read_input_bulk( rec = (n * INPUT_RECORD)() read = DWORD() if not ReadConsoleInput(InHandle, rec, n, read): - raise WinError(GetLastError()) + raise WinError(get_last_error()) return rec, read.value @@ -512,15 +634,17 @@ def beep(self) -> None: def clear(self) -> None: """Wipe the screen""" + trace("windows.clear") self.__write(CLEAR) self.posxy = 0, 0 - self.screen = [""] + self.sync_rendered_screen(RenderedScreen.empty(), self.posxy) def finish(self) -> None: """Move the cursor to the end of the display and otherwise get ready for end. XXX could be merged with restore? Hmm.""" - y = len(self.screen) - 1 - while y >= 0 and not self.screen[y]: + rendered_lines = self._rendered_screen.composed_lines + y = len(rendered_lines) - 1 + while y >= 0 and not rendered_lines[y].text: y -= 1 self._move_relative(0, min(y, self.height + self.__offset - 1)) self.__write("\r\n") @@ -535,7 +659,7 @@ def flushoutput(self) -> None: def forgetinput(self) -> None: """Forget all pending, but not yet processed input.""" if not FlushConsoleInputBuffer(InHandle): - raise WinError(GetLastError()) + raise WinError(get_last_error()) def getpending(self) -> Event: """Return the characters that have been typed but not yet @@ -562,11 +686,11 @@ def getpending(self) -> Event: # ignore SHIFT_PRESSED and special keys continue if ch == "\r": - ch += "\n" + ch = "\n" e.data += ch return e - def wait(self, timeout: float | None) -> bool: + def wait_for_event(self, timeout: float | None) -> bool: """Wait for an event.""" if timeout is None: timeout = INFINITE @@ -579,7 +703,17 @@ def wait(self, timeout: float | None) -> bool: return False return True + def wait(self, timeout: float | None) -> bool: + """ + Wait for events on the console. + """ + return ( + not self.event_queue.empty() + or self.wait_for_event(timeout) + ) + def repaint(self) -> None: + trace("windows.repaint unsupported") raise NotImplementedError("No repaint support") diff --git a/Lib/_sitebuiltins.py b/Lib/_sitebuiltins.py index c66269a5719..84551e3546e 100644 --- a/Lib/_sitebuiltins.py +++ b/Lib/_sitebuiltins.py @@ -36,7 +36,7 @@ def __init__(self, name, data, files=(), dirs=()): import os self.__name = name self.__data = data - self.__lines = None + self.__lines = [] self.__filenames = [os.path.join(dir, filename) for dir in dirs for filename in files] @@ -65,24 +65,22 @@ def __repr__(self): return "Type %s() to see the full %s text" % ((self.__name,)*2) def __call__(self): - self.__setup() - prompt = 'Hit Return for more, or q (and Return) to quit: ' - lineno = 0 - while 1: + try: + from _pyrepl.pager import get_pager + except ModuleNotFoundError: try: - for i in range(lineno, lineno + self.MAXLINES): - print(self.__lines[i]) - except IndexError: - break - else: - lineno += self.MAXLINES - key = None - while key is None: - key = input(prompt) - if key not in ('', 'q'): - key = None - if key == 'q': - break + from pydoc import get_pager + except ModuleNotFoundError: + def get_pager(): + def _print(text, title=None): + print(text) + return _print + + self.__setup() + + pager = get_pager() + text = "\n".join(self.__lines) + pager(text, title=self.__name) class _Helper(object): diff --git a/Lib/_strptime.py b/Lib/_strptime.py index cdc55e8daaf..746b0907c1d 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -7,7 +7,7 @@ FUNCTIONS: _getlang -- Figure out what language is being used for the locale - strptime -- Calculates the time struct represented by the passed-in string + _strptime -- Calculates the time struct represented by the passed-in string """ import os @@ -371,7 +371,9 @@ def __init__(self, locale_time=None): # W is set below by using 'U' 'y': r"(?P\d\d)", 'Y': r"(?P\d\d\d\d)", + # See gh-121237: "z" must support colons for backwards compatibility. 'z': r"(?P([+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?)|(?-i:Z))?", + ':z': r"(?P([+-]\d\d:[0-5]\d(:[0-5]\d(\.\d{1,6})?)?)|(?-i:Z))?", 'A': self.__seqToRE(self.locale_time.f_weekday, 'A'), 'a': self.__seqToRE(self.locale_time.a_weekday, 'a'), 'B': self.__seqToRE(_fixmonths(self.locale_time.f_month[1:]), 'B'), @@ -380,7 +382,10 @@ def __init__(self, locale_time=None): 'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone for tz in tz_names), 'Z'), - '%': '%'} + 'n': r'\s*', + 't': r'\s*', + '%': '%', + } if self.locale_time.LC_alt_digits is None: for d in 'dmyCHIMS': mapping['O' + d] = r'(?P<%s>\d\d|\d| \d)' % d @@ -416,6 +421,8 @@ def __init__(self, locale_time=None): mapping['W'] = mapping['U'].replace('U', 'W') base.__init__(mapping) + base.__setitem__('D', self.pattern('%m/%d/%y')) + base.__setitem__('F', self.pattern('%Y-%m-%d')) base.__setitem__('T', self.pattern('%H:%M:%S')) base.__setitem__('R', self.pattern('%H:%M')) base.__setitem__('r', self.pattern(self.locale_time.LC_time_ampm)) @@ -457,28 +464,39 @@ def pattern(self, format): format = re_sub(r'\s+', r'\\s+', format) format = re_sub(r"'", "['\u02bc]", format) # needed for br_FR year_in_format = False - day_of_month_in_format = False + day_d_in_format = False + day_e_in_format = False def repl(m): - format_char = m[1] - match format_char: + directive = m.group()[1:] # exclude `%` symbol + match directive: case 'Y' | 'y' | 'G': nonlocal year_in_format year_in_format = True case 'd': - nonlocal day_of_month_in_format - day_of_month_in_format = True - return self[format_char] - format = re_sub(r'%[-_0^#]*[0-9]*([OE]?\\?.?)', repl, format) - if day_of_month_in_format and not year_in_format: - import warnings - warnings.warn("""\ + nonlocal day_d_in_format + day_d_in_format = True + case 'e': + nonlocal day_e_in_format + day_e_in_format = True + return self[directive] + format = re_sub(r'%[-_0^#]*[0-9]*([OE]?[:\\]?.?)', repl, format) + if not year_in_format: + if day_d_in_format: + raise ValueError( + "Day of month directive '%d' may not be used without " + "a year directive. Parsing dates involving a day of " + "month without a year is ambiguous and fails to parse " + "leap day. Add a year to the input and format. " + "See https://github.com/python/cpython/issues/70647.") + if day_e_in_format: + import warnings + warnings.warn("""\ Parsing dates involving a day of month without a year specified is ambiguous -and fails to parse leap day. The default behavior will change in Python 3.15 -to either always raise an exception or to use a different default year (TBD). -To avoid trouble, add a specific year to the input & format. +and fails to parse leap day. '%e' without a year will become an error in Python 3.17. +To avoid trouble, add a specific year to the input and format. See https://github.com/python/cpython/issues/70647.""", - DeprecationWarning, - skip_file_prefixes=(os.path.dirname(__file__),)) + DeprecationWarning, + skip_file_prefixes=(os.path.dirname(__file__),)) return format def compile(self, format): @@ -514,9 +532,10 @@ def _calc_julian_from_U_or_W(year, week_of_year, day_of_week, week_starts_Mon): def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): - """Return a 2-tuple consisting of a time struct and an int containing - the number of microseconds based on the input string and the - format string.""" + """Return a 3-tuple consisting of a tuple with time components, + an int containing the number of microseconds, and an int + containing the microseconds part of the GMT offset, based on the + input string and the format string.""" for index, arg in enumerate([data_string, format]): if not isinstance(arg, str): @@ -555,8 +574,17 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): raise ValueError("time data %r does not match format %r" % (data_string, format)) if len(data_string) != found.end(): - raise ValueError("unconverted data remains: %s" % - data_string[found.end():]) + rest = data_string[found.end():] + # Specific check for '%:z' directive + if ( + "colon_z" in found.re.groupindex + and found.group("colon_z") is not None + and rest[0] != ":" + ): + raise ValueError( + f"Missing colon in %:z before '{rest}', got '{data_string}'" + ) + raise ValueError("unconverted data remains: %s" % rest) iso_year = year = None month = day = 1 @@ -616,18 +644,18 @@ def parse_int(s): hour = parse_int(found_dict['I']) ampm = found_dict.get('p', '').lower() # If there was no AM/PM indicator, we'll treat this like AM - if ampm in ('', locale_time.am_pm[0]): - # We're in AM so the hour is correct unless we're - # looking at 12 midnight. - # 12 midnight == 12 AM == hour 0 - if hour == 12: - hour = 0 - elif ampm == locale_time.am_pm[1]: + if ampm == locale_time.am_pm[1]: # We're in PM so we need to add 12 to the hour unless # we're looking at 12 noon. # 12 noon == 12 PM == hour 12 if hour != 12: hour += 12 + else: + # We're in AM so the hour is correct unless we're + # looking at 12 midnight. + # 12 midnight == 12 AM == hour 0 + if hour == 12: + hour = 0 elif group_key == 'M': minute = parse_int(found_dict['M']) elif group_key == 'S': @@ -662,8 +690,8 @@ def parse_int(s): week_of_year_start = 0 elif group_key == 'V': iso_week = int(found_dict['V']) - elif group_key == 'z': - z = found_dict['z'] + elif group_key in ('z', 'colon_z'): + z = found_dict[group_key] if z: if z == 'Z': gmtoff = 0 @@ -672,7 +700,7 @@ def parse_int(s): z = z[:3] + z[4:] if len(z) > 5: if z[5] != ':': - msg = f"Inconsistent use of : in {found_dict['z']}" + msg = f"Inconsistent use of : in {found_dict[group_key]}" raise ValueError(msg) z = z[:5] + z[6:] hours = int(z[1:3]) diff --git a/Lib/_threading_local.py b/Lib/_threading_local.py index 0b9e5d3bbf6..2af3885458b 100644 --- a/Lib/_threading_local.py +++ b/Lib/_threading_local.py @@ -57,7 +57,7 @@ def thread_deleted(_, idt=idt): # as soon as the OS-level thread ends instead. local = wrlocal() if local is not None: - dct = local.dicts.pop(idt) + local.dicts.pop(idt) wrlocal = ref(self, local_deleted) wrthread = ref(thread, thread_deleted) thread.__dict__[key] = wrlocal diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index bee019cd515..5c9a0812646 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -47,6 +47,7 @@ class Format(enum.IntEnum): "__cell__", "__owner__", "__stringifier_dict__", + "__resolved_str_cache__", ) @@ -85,12 +86,16 @@ def __init__( # These are always set to None here but may be non-None if a ForwardRef # is created through __class__ assignment on a _Stringifier object. self.__globals__ = None + # This may be either a cell object (for a ForwardRef referring to a single name) + # or a dict mapping cell names to cell objects (for a ForwardRef containing references + # to multiple names). self.__cell__ = None self.__extra_names__ = None # These are initially None but serve as a cache and may be set to a non-None # value later. self.__code__ = None self.__ast_node__ = None + self.__resolved_str_cache__ = None def __init_subclass__(cls, /, *args, **kwds): raise TypeError("Cannot subclass ForwardRef") @@ -110,14 +115,14 @@ def evaluate( """ match format: case Format.STRING: - return self.__forward_arg__ + return self.__resolved_str__ case Format.VALUE: is_forwardref_format = False case Format.FORWARDREF: is_forwardref_format = True case _: raise NotImplementedError(format) - if self.__cell__ is not None: + if isinstance(self.__cell__, types.CellType): try: return self.__cell__.cell_contents except ValueError: @@ -147,26 +152,42 @@ def evaluate( if globals is None: globals = {} + if type_params is None and owner is not None: + type_params = getattr(owner, "__type_params__", None) + if locals is None: locals = {} if isinstance(owner, type): locals.update(vars(owner)) + elif ( + type_params is not None + or isinstance(self.__cell__, dict) + or self.__extra_names__ + ): + # Create a new locals dict if necessary, + # to avoid mutating the argument. + locals = dict(locals) - if type_params is None and owner is not None: - # "Inject" type parameters into the local namespace - # (unless they are shadowed by assignments *in* the local namespace), - # as a way of emulating annotation scopes when calling `eval()` - type_params = getattr(owner, "__type_params__", None) - - # Type parameters exist in their own scope, which is logically - # between the locals and the globals. We simulate this by adding - # them to the globals. + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` if type_params is not None: - globals = dict(globals) for param in type_params: - globals[param.__name__] = param + locals.setdefault(param.__name__, param) + + # Similar logic can be used for nonlocals, which should not + # override locals. + if isinstance(self.__cell__, dict): + for cell_name, cell in self.__cell__.items(): + try: + cell_value = cell.cell_contents + except ValueError: + pass + else: + locals.setdefault(cell_name, cell_value) + if self.__extra_names__: - locals = {**locals, **self.__extra_names__} + locals.update(self.__extra_names__) arg = self.__forward_arg__ if arg.isidentifier() and not keyword.iskeyword(arg): @@ -187,8 +208,11 @@ def evaluate( except Exception: if not is_forwardref_format: raise + + # All variables, in scoping order, should be checked before + # triggering __missing__ to create a _Stringifier. new_locals = _StringifierDict( - {**builtins.__dict__, **locals}, + {**builtins.__dict__, **globals, **locals}, globals=globals, owner=owner, is_class=self.__forward_is_class__, @@ -199,7 +223,7 @@ def evaluate( except Exception: return self else: - new_locals.transmogrify() + new_locals.transmogrify(self.__cell__) return result def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): @@ -236,20 +260,31 @@ def __forward_arg__(self): "Attempted to access '__forward_arg__' on an uninitialized ForwardRef" ) + @property + def __resolved_str__(self): + # __forward_arg__ with any names from __extra_names__ replaced + # with the type_repr of the value they represent + if self.__resolved_str_cache__ is None: + resolved_str = self.__forward_arg__ + names = self.__extra_names__ + + if names: + visitor = _ExtraNameFixer(names) + ast_expr = ast.parse(resolved_str, mode="eval").body + node = visitor.visit(ast_expr) + resolved_str = ast.unparse(node) + + self.__resolved_str_cache__ = resolved_str + + return self.__resolved_str_cache__ + @property def __forward_code__(self): if self.__code__ is not None: return self.__code__ arg = self.__forward_arg__ - # If we do `def f(*args: *Ts)`, then we'll have `arg = '*Ts'`. - # Unfortunately, this isn't a valid expression on its own, so we - # do the unpacking manually. - if arg.startswith("*"): - arg_to_compile = f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] - else: - arg_to_compile = arg try: - self.__code__ = compile(arg_to_compile, "", "eval") + self.__code__ = compile(_rewrite_star_unpack(arg), "", "eval") except SyntaxError: raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") return self.__code__ @@ -264,7 +299,13 @@ def __eq__(self, other): # because dictionaries are not hashable. and self.__globals__ is other.__globals__ and self.__forward_is_class__ == other.__forward_is_class__ - and self.__cell__ == other.__cell__ + # Two separate cells are always considered unequal in forward refs. + and ( + {name: id(cell) for name, cell in self.__cell__.items()} + == {name: id(cell) for name, cell in other.__cell__.items()} + if isinstance(self.__cell__, dict) and isinstance(other.__cell__, dict) + else self.__cell__ is other.__cell__ + ) and self.__owner__ == other.__owner__ and ( (tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None) == @@ -278,7 +319,10 @@ def __hash__(self): self.__forward_module__, id(self.__globals__), # dictionaries are not hashable, so hash by identity self.__forward_is_class__, - self.__cell__, + ( # cells are not hashable as well + tuple(sorted([(name, id(cell)) for name, cell in self.__cell__.items()])) + if isinstance(self.__cell__, dict) else id(self.__cell__), + ), self.__owner__, tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None, )) @@ -297,7 +341,7 @@ def __repr__(self): extra.append(", is_class=True") if self.__owner__ is not None: extra.append(f", owner={self.__owner__!r}") - return f"ForwardRef({self.__forward_arg__!r}{''.join(extra)})" + return f"ForwardRef({self.__resolved_str__!r}{''.join(extra)})" _Template = type(t"") @@ -333,6 +377,7 @@ def __init__( self.__cell__ = cell self.__owner__ = owner self.__stringifier_dict__ = stringifier_dict + self.__resolved_str_cache__ = None # Needed for ForwardRef def __convert_to_ast(self, other): if isinstance(other, _Stringifier): @@ -560,32 +605,70 @@ def unary_op(self): del _make_unary_op -def _template_to_ast(template): +def _template_to_ast_constructor(template): + """Convert a `template` instance to a non-literal AST.""" + args = [] + for part in template: + match part: + case str(): + args.append(ast.Constant(value=part)) + case _: + interp = ast.Call( + func=ast.Name(id="Interpolation"), + args=[ + ast.Constant(value=part.value), + ast.Constant(value=part.expression), + ast.Constant(value=part.conversion), + ast.Constant(value=part.format_spec), + ] + ) + args.append(interp) + return ast.Call(func=ast.Name(id="Template"), args=args, keywords=[]) + + +def _template_to_ast_literal(template, parsed): + """Convert a `template` instance to a t-string literal AST.""" values = [] + interp_count = 0 for part in template: match part: case str(): values.append(ast.Constant(value=part)) - # Interpolation, but we don't want to import the string module case _: interp = ast.Interpolation( str=part.expression, - value=ast.parse(part.expression), - conversion=( - ord(part.conversion) - if part.conversion is not None - else -1 - ), - format_spec=( - ast.Constant(value=part.format_spec) - if part.format_spec != "" - else None - ), + value=parsed[interp_count], + conversion=ord(part.conversion) if part.conversion else -1, + format_spec=ast.Constant(value=part.format_spec) + if part.format_spec + else None, ) values.append(interp) + interp_count += 1 return ast.TemplateStr(values=values) +def _template_to_ast(template): + """Make a best-effort conversion of a `template` instance to an AST.""" + # gh-138558: Not all Template instances can be represented as t-string + # literals. Return the most accurate AST we can. See issue for details. + + # If any expr is empty or whitespace only, we cannot convert to a literal. + if any(part.expression.strip() == "" for part in template.interpolations): + return _template_to_ast_constructor(template) + + try: + # Wrap in parens to allow whitespace inside interpolation curly braces + parsed = tuple( + ast.parse(f"({part.expression})", mode="eval").body + for part in template.interpolations + ) + except SyntaxError: + return _template_to_ast_constructor(template) + + return _template_to_ast_literal(template, parsed) + + class _StringifierDict(dict): def __init__(self, namespace, *, globals=None, owner=None, is_class=False, format): super().__init__(namespace) @@ -608,13 +691,15 @@ def __missing__(self, key): self.stringifiers.append(fwdref) return fwdref - def transmogrify(self): + def transmogrify(self, cell_dict): for obj in self.stringifiers: obj.__class__ = ForwardRef obj.__stringifier_dict__ = None # not needed for ForwardRef if isinstance(obj.__ast_node__, str): obj.__arg__ = obj.__ast_node__ obj.__ast_node__ = None + if cell_dict is not None and obj.__cell__ is None: + obj.__cell__ = cell_dict def create_unique_name(self): name = f"__annotationlib_name_{self.next_id}__" @@ -664,9 +749,21 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): # possibly constants if the annotate function uses them directly). We then # convert each of those into a string to get an approximation of the # original source. + + # Attempt to call with VALUE_WITH_FAKE_GLOBALS to check if it is implemented + # See: https://github.com/python/cpython/issues/138764 + # Only fail on NotImplementedError + try: + annotate(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # Both STRING and VALUE_WITH_FAKE_GLOBALS are not implemented: fallback to VALUE + return annotations_to_string(annotate(Format.VALUE)) + except Exception: + pass + globals = _StringifierDict({}, format=format) is_class = isinstance(owner, type) - closure = _build_closure( + closure, _ = _build_closure( annotate, owner, is_class, globals, allow_evaluation=False ) func = types.FunctionType( @@ -710,7 +807,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): is_class=is_class, format=format, ) - closure = _build_closure( + closure, cell_dict = _build_closure( annotate, owner, is_class, globals, allow_evaluation=True ) func = types.FunctionType( @@ -722,10 +819,13 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): ) try: result = func(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # FORWARDREF and VALUE_WITH_FAKE_GLOBALS not supported, fall back to VALUE + return annotate(Format.VALUE) except Exception: pass else: - globals.transmogrify() + globals.transmogrify(cell_dict) return result # Try again, but do not provide any globals. This allows us to return @@ -737,7 +837,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): is_class=is_class, format=format, ) - closure = _build_closure( + closure, cell_dict = _build_closure( annotate, owner, is_class, globals, allow_evaluation=False ) func = types.FunctionType( @@ -748,7 +848,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): kwdefaults=annotate.__kwdefaults__, ) result = func(Format.VALUE_WITH_FAKE_GLOBALS) - globals.transmogrify() + globals.transmogrify(cell_dict) if _is_evaluate: if isinstance(result, ForwardRef): return result.evaluate(format=Format.FORWARDREF) @@ -773,14 +873,11 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluation): if not annotate.__closure__: - return None - freevars = annotate.__code__.co_freevars + return None, None new_closure = [] - for i, cell in enumerate(annotate.__closure__): - if i < len(freevars): - name = freevars[i] - else: - name = "__cell__" + cell_dict = {} + for name, cell in zip(annotate.__code__.co_freevars, annotate.__closure__, strict=True): + cell_dict[name] = cell new_cell = None if allow_evaluation: try: @@ -801,7 +898,7 @@ def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluat stringifier_dict.stringifiers.append(fwdref) new_cell = types.CellType(fwdref) new_closure.append(new_cell) - return tuple(new_closure) + return tuple(new_closure), cell_dict def _stringify_single(anno): @@ -843,7 +940,7 @@ def get_annotations( does not exist, the __annotate__ function is called. The FORWARDREF format uses __annotations__ if it exists and can be evaluated, and otherwise falls back to calling the __annotate__ function. - The SOURCE format tries __annotate__ first, and falls back to + The STRING format tries __annotate__ first, and falls back to using __annotations__, stringified using annotations_to_string(). This function handles several details for you: @@ -961,13 +1058,26 @@ def get_annotations( obj_globals = obj_locals = unwrap = None if unwrap is not None: + # Use an id-based visited set to detect cycles in the __wrapped__ + # and functools.partial.func chain (e.g. f.__wrapped__ = f). + # On cycle detection we stop and use whatever __globals__ we have + # found so far, mirroring the approach of inspect.unwrap(). + _seen_ids = {id(unwrap)} while True: if hasattr(unwrap, "__wrapped__"): - unwrap = unwrap.__wrapped__ + candidate = unwrap.__wrapped__ + if id(candidate) in _seen_ids: + break + _seen_ids.add(id(candidate)) + unwrap = candidate continue if functools := sys.modules.get("functools"): if isinstance(unwrap, functools.partial): - unwrap = unwrap.func + candidate = unwrap.func + if id(candidate) in _seen_ids: + break + _seen_ids.add(id(candidate)) + unwrap = candidate continue break if hasattr(unwrap, "__globals__"): @@ -987,7 +1097,8 @@ def get_annotations( locals = {param.__name__: param for param in type_params} | locals return_value = { - key: value if not isinstance(value, str) else eval(value, globals, locals) + key: value if not isinstance(value, str) + else eval(_rewrite_star_unpack(value), globals, locals) for key, value in ann.items() } return return_value @@ -1024,6 +1135,16 @@ def annotations_to_string(annotations): } +def _rewrite_star_unpack(arg): + """If the given argument annotation expression is a star unpack e.g. `'*Ts'` + rewrite it to a valid expression. + """ + if arg.lstrip().startswith("*"): + return f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] + else: + return arg + + def _get_and_call_annotate(obj, format): """Get the __annotate__ function and call it. @@ -1063,3 +1184,14 @@ def _get_dunder_annotations(obj): if not isinstance(ann, dict): raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") return ann + + +class _ExtraNameFixer(ast.NodeTransformer): + """Fixer for __extra_names__ items in ForwardRef __repr__ and string evaluation""" + def __init__(self, extra_names): + self.extra_names = extra_names + + def visit_Name(self, node: ast.Name): + if (new_name := self.extra_names.get(node.id, _sentinel)) is not _sentinel: + node = ast.Name(id=type_repr(new_name)) + return node diff --git a/Lib/argparse.py b/Lib/argparse.py index 2144c81886a..d91707d9eec 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -64,7 +64,6 @@ still considered an implementation detail.) """ -__version__ = '1.1' __all__ = [ 'ArgumentParser', 'ArgumentError', @@ -90,8 +89,8 @@ import os as _os import re as _re import sys as _sys - -from gettext import gettext as _, ngettext +from gettext import gettext as _ +from gettext import ngettext SUPPRESS = '==SUPPRESS==' @@ -149,6 +148,10 @@ def _copy_items(items): return copy.copy(items) +def _identity(value): + return value + + # =============== # Formatting Help # =============== @@ -167,7 +170,6 @@ def __init__( indent_increment=2, max_help_position=24, width=None, - color=True, ): # default setting for width if width is None: @@ -175,7 +177,6 @@ def __init__( width = shutil.get_terminal_size().columns width -= 2 - self._set_color(color) self._prog = prog self._indent_increment = indent_increment self._max_help_position = min(max_help_position, @@ -192,15 +193,17 @@ def __init__( self._whitespace_matcher = _re.compile(r'\s+', _re.ASCII) self._long_break_matcher = _re.compile(r'\n\n\n+') - def _set_color(self, color): + self._set_color(False) + + def _set_color(self, color, *, file=None): from _colorize import can_colorize, decolor, get_theme - if color and can_colorize(): + if color and can_colorize(file=file): self._theme = get_theme(force_color=True).argparse self._decolor = decolor else: self._theme = get_theme(force_no_color=True).argparse - self._decolor = lambda text: text + self._decolor = _identity # =============================== # Section and indentation methods @@ -281,7 +284,7 @@ def add_argument(self, action): if action.help is not SUPPRESS: # find all invocations - get_invocation = self._format_action_invocation + get_invocation = lambda x: self._decolor(self._format_action_invocation(x)) invocation_lengths = [len(get_invocation(action)) + self._current_indent] for subaction in self._iter_indented_subactions(action): invocation_lengths.append(len(get_invocation(subaction)) + self._current_indent) @@ -337,27 +340,17 @@ def _format_usage(self, usage, actions, groups, prefix): elif usage is None: prog = '%(prog)s' % dict(prog=self._prog) - # split optionals from positionals - optionals = [] - positionals = [] - for action in actions: - if action.option_strings: - optionals.append(action) - else: - positionals.append(action) - + parts, pos_start = self._get_actions_usage_parts(actions, groups) # build full usage string - format = self._format_actions_usage - action_usage = format(optionals + positionals, groups) - usage = ' '.join([s for s in [prog, action_usage] if s]) + usage = ' '.join(filter(None, [prog, *parts])) # wrap the usage parts if it's too long text_width = self._width - self._current_indent if len(prefix) + len(self._decolor(usage)) > text_width: # break usage into wrappable parts - opt_parts = self._get_actions_usage_parts(optionals, groups) - pos_parts = self._get_actions_usage_parts(positionals, groups) + opt_parts = parts[:pos_start] + pos_parts = parts[pos_start:] # helper for wrapping lines def get_lines(parts, indent, prefix=None): @@ -414,117 +407,141 @@ def get_lines(parts, indent, prefix=None): # prefix with 'usage:' return f'{t.usage}{prefix}{t.reset}{usage}\n\n' - def _format_actions_usage(self, actions, groups): - return ' '.join(self._get_actions_usage_parts(actions, groups)) - def _is_long_option(self, string): return len(string) > 2 def _get_actions_usage_parts(self, actions, groups): - # find group indices and identify actions in groups - group_actions = set() - inserts = {} + """Get usage parts with split index for optionals/positionals. + + Returns (parts, pos_start) where pos_start is the index in parts + where positionals begin. + This preserves mutually exclusive group formatting across the + optionals/positionals boundary (gh-75949). + """ + actions = [action for action in actions if action.help is not SUPPRESS] + # group actions by mutually exclusive groups + action_groups = dict.fromkeys(actions) for group in groups: - if not group._group_actions: - raise ValueError(f'empty group {group}') - - if all(action.help is SUPPRESS for action in group._group_actions): - continue - - try: - start = min(actions.index(item) for item in group._group_actions) - except ValueError: - continue - else: - end = start + len(group._group_actions) - if set(actions[start:end]) == set(group._group_actions): - group_actions.update(group._group_actions) - inserts[start, end] = group + for action in group._group_actions: + if action in action_groups: + action_groups[action] = group + # positional arguments keep their position + positionals = [] + for action in actions: + if not action.option_strings: + group = action_groups.pop(action) + if group: + group_actions = [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + [action] + positionals.append((group.required, group_actions)) + else: + positionals.append((None, [action])) + # the remaining optional arguments are sorted by the position of + # the first option in the group + optionals = [] + for action in actions: + if action.option_strings and action in action_groups: + group = action_groups.pop(action) + if group: + group_actions = [action] + [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + optionals.append((group.required, group_actions)) + else: + optionals.append((None, [action])) # collect all actions format strings parts = [] t = self._theme - for action in actions: + pos_start = None + for i, (required, group) in enumerate(optionals + positionals): + start = len(parts) + if i == len(optionals): + pos_start = start + in_group = len(group) > 1 + for action in group: + # produce all arg strings + if not action.option_strings: + default = self._get_default_metavar_for_positional(action) + part = self._format_args(action, default) + # if it's in a group, strip the outer [] + if in_group: + if part[0] == '[' and part[-1] == ']': + part = part[1:-1] + part = t.summary_action + part + t.reset - # suppressed arguments are marked with None - if action.help is SUPPRESS: - part = None - - # produce all arg strings - elif not action.option_strings: - default = self._get_default_metavar_for_positional(action) - part = ( - t.summary_action - + self._format_args(action, default) - + t.reset - ) - - # if it's in a group, strip the outer [] - if action in group_actions: - if part[0] == '[' and part[-1] == ']': - part = part[1:-1] - - # produce the first way to invoke the option in brackets - else: - option_string = action.option_strings[0] - if self._is_long_option(option_string): - option_color = t.summary_long_option + # produce the first way to invoke the option in brackets else: - option_color = t.summary_short_option + option_string = action.option_strings[0] + if self._is_long_option(option_string): + option_color = t.summary_long_option + else: + option_color = t.summary_short_option - # if the Optional doesn't take a value, format is: - # -s or --long - if action.nargs == 0: - part = action.format_usage() - part = f"{option_color}{part}{t.reset}" + # if the Optional doesn't take a value, format is: + # -s or --long + if action.nargs == 0: + part = action.format_usage() + part = f"{option_color}{part}{t.reset}" - # if the Optional takes a value, format is: - # -s ARGS or --long ARGS - else: - default = self._get_default_metavar_for_optional(action) - args_string = self._format_args(action, default) - part = ( - f"{option_color}{option_string} " - f"{t.summary_label}{args_string}{t.reset}" - ) + # if the Optional takes a value, format is: + # -s ARGS or --long ARGS + else: + default = self._get_default_metavar_for_optional(action) + args_string = self._format_args(action, default) + part = ( + f"{option_color}{option_string} " + f"{t.summary_label}{args_string}{t.reset}" + ) - # make it look optional if it's not required or in a group - if not action.required and action not in group_actions: - part = '[%s]' % part + # make it look optional if it's not required or in a group + if not (action.required or required or in_group): + part = '[%s]' % part - # add the action string to the list - parts.append(part) + # add the action string to the list + parts.append(part) - # group mutually exclusive actions - inserted_separators_indices = set() - for start, end in sorted(inserts, reverse=True): - group = inserts[start, end] - group_parts = [item for item in parts[start:end] if item is not None] - group_size = len(group_parts) - if group.required: - open, close = "()" if group_size > 1 else ("", "") - else: - open, close = "[]" - group_parts[0] = open + group_parts[0] - group_parts[-1] = group_parts[-1] + close - for i, part in enumerate(group_parts[:-1], start=start): - # insert a separator if not already done in a nested group - if i not in inserted_separators_indices: - parts[i] = part + ' |' - inserted_separators_indices.add(i) - parts[start + group_size - 1] = group_parts[-1] - for i in range(start + group_size, end): - parts[i] = None + if in_group: + parts[start] = ('(' if required else '[') + parts[start] + for i in range(start, len(parts) - 1): + parts[i] += ' |' + parts[-1] += ')' if required else ']' - # return the usage parts - return [item for item in parts if item is not None] + if pos_start is None: + pos_start = len(parts) + return parts, pos_start def _format_text(self, text): if '%(prog)' in text: text = text % dict(prog=self._prog) text_width = max(self._width - self._current_indent, 11) indent = ' ' * self._current_indent - return self._fill_text(text, text_width, indent) + '\n\n' + text = self._fill_text(text, text_width, indent) + text = self._apply_text_markup(text) + return text + '\n\n' + + def _apply_text_markup(self, text): + """Apply color markup to text. + + Supported markup: + `...` - inline code (rendered with prog_extra color) + + When colors are disabled, backticks are preserved as-is. + """ + t = self._theme + if not t.reset: + return text + text = _re.sub( + r'`([^`]+)`', + rf'{t.prog_extra}\1{t.reset}', + text, + ) + return text def _format_action(self, action): # determine the required width and the entry label @@ -675,7 +692,41 @@ def _expand_help(self, action): params[name] = value.__name__ if params.get('choices') is not None: params['choices'] = ', '.join(map(str, params['choices'])) - return help_string % params + + t = self._theme + + result = help_string % params + + if not t.reset: + return result + + # Match format specifiers like: %s, %d, %(key)s, etc. + fmt_spec = r''' + % + (?: + % # %% escape + | + (?:\((?P[^)]*)\))? # key + [-#0\ +]* # flags + (?:\*|\d+)? # width + (?:\.(?:\*|\d+))? # precision + [hlL]? # length modifier + [diouxXeEfFgGcrsa] # conversion type + ) + ''' + + def colorize(match): + spec, key = match.group(0, 'key') + if spec == '%%': + return '%' + if key is not None: + # %(key)... - format and colorize + formatted = spec % {key: params[key]} + return f'{t.interpolated_value}{formatted}{t.reset}' + # bare %s etc. - format with full params dict, no colorization + return spec % params + + return _re.sub(fmt_spec, colorize, help_string, flags=_re.VERBOSE) def _iter_indented_subactions(self, action): try: @@ -745,11 +796,21 @@ def _get_help_string(self, action): if help is None: help = '' - if '%(default)' not in help: - if action.default is not SUPPRESS: - defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] - if action.option_strings or action.nargs in defaulting_nargs: - help += _(' (default: %(default)s)') + if ( + '%(default)' not in help + and action.default is not SUPPRESS + and not action.required + ): + defaulting_nargs = (OPTIONAL, ZERO_OR_MORE) + if action.option_strings or action.nargs in defaulting_nargs: + t = self._theme + default_str = _(" (default: %(default)s)") + prefix, suffix = default_str.split("%(default)s") + help += ( + f" {t.default}{prefix.lstrip()}{t.reset}" + f"%(default)s" + f"{t.default}{suffix}{t.reset}" + ) return help @@ -933,15 +994,26 @@ def __init__(self, deprecated=False): _option_strings = [] + neg_option_strings = [] for option_string in option_strings: _option_strings.append(option_string) - if option_string.startswith('--'): - if option_string.startswith('--no-'): + if len(option_string) > 2 and option_string[0] == option_string[1]: + # two-dash long option: '--foo' -> '--no-foo' + if option_string.startswith('no-', 2): raise ValueError(f'invalid option name {option_string!r} ' f'for BooleanOptionalAction') - option_string = '--no-' + option_string[2:] + option_string = option_string[:2] + 'no-' + option_string[2:] _option_strings.append(option_string) + neg_option_strings.append(option_string) + elif len(option_string) > 2 and option_string[0] != option_string[1]: + # single-dash long option: '-foo' -> '-nofoo' + if option_string.startswith('no', 1): + raise ValueError(f'invalid option name {option_string!r} ' + f'for BooleanOptionalAction') + option_string = option_string[:1] + 'no' + option_string[1:] + _option_strings.append(option_string) + neg_option_strings.append(option_string) super().__init__( option_strings=_option_strings, @@ -951,11 +1023,12 @@ def __init__(self, required=required, help=help, deprecated=deprecated) + self.neg_option_strings = neg_option_strings def __call__(self, parser, namespace, values, option_string=None): if option_string in self.option_strings: - setattr(namespace, self.dest, not option_string.startswith('--no-')) + setattr(namespace, self.dest, option_string not in self.neg_option_strings) def format_usage(self): return ' | '.join(self.option_strings) @@ -1552,8 +1625,8 @@ def add_argument(self, *args, **kwargs): f'instance of it must be passed') # raise an error if the metavar does not match the type - if hasattr(self, "_get_formatter"): - formatter = self._get_formatter() + if hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: formatter._format_args(action, None) except TypeError: @@ -1661,29 +1734,35 @@ def _get_positional_kwargs(self, dest, **kwargs): def _get_optional_kwargs(self, *args, **kwargs): # determine short and long option strings option_strings = [] - long_option_strings = [] for option_string in args: # error on strings that don't start with an appropriate prefix - if not option_string[0] in self.prefix_chars: + if option_string[0] not in self.prefix_chars: raise ValueError( f'invalid option string {option_string!r}: ' f'must start with a character {self.prefix_chars!r}') - - # strings starting with two prefix characters are long options option_strings.append(option_string) - if len(option_string) > 1 and option_string[1] in self.prefix_chars: - long_option_strings.append(option_string) # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x' dest = kwargs.pop('dest', None) if dest is None: - if long_option_strings: - dest_option_string = long_option_strings[0] - else: - dest_option_string = option_strings[0] - dest = dest_option_string.lstrip(self.prefix_chars) + priority = 0 + for option_string in option_strings: + if len(option_string) <= 2: + # short option: '-x' -> 'x' + if priority < 1: + dest = option_string.lstrip(self.prefix_chars) + priority = 1 + elif option_string[1] not in self.prefix_chars: + # single-dash long option: '-foo' -> 'foo' + if priority < 2: + dest = option_string.lstrip(self.prefix_chars) + priority = 2 + else: + # two-dash long option: '--foo' -> 'foo' + dest = option_string.lstrip(self.prefix_chars) + break if not dest: - msg = f'dest= is required for options like {option_string!r}' + msg = f'dest= is required for options like {repr(option_strings)[1:-1]}' raise TypeError(msg) dest = dest.replace('-', '_') @@ -1741,8 +1820,8 @@ def _handle_conflict_resolve(self, action, conflicting_actions): action.container._remove_action(action) def _check_help(self, action): - if action.help and hasattr(self, "_get_formatter"): - formatter = self._get_formatter() + if action.help and hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: formatter._expand_help(action) except (ValueError, TypeError, KeyError) as exc: @@ -1858,7 +1937,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer): - exit_on_error -- Determines whether or not ArgumentParser exits with error info when an error occurs - suggest_on_error - Enables suggestions for mistyped argument choices - and subparser names (default: ``False``) + and subparser names (default: ``True``) - color - Allow color output in help messages (default: ``False``) """ @@ -1877,7 +1956,7 @@ def __init__(self, allow_abbrev=True, exit_on_error=True, *, - suggest_on_error=False, + suggest_on_error=True, color=True, ): superinit = super(ArgumentParser, self).__init__ @@ -1897,15 +1976,16 @@ def __init__(self, self.suggest_on_error = suggest_on_error self.color = color + # Cached formatter for validation (avoids repeated _set_color calls) + self._cached_formatter = None + add_group = self.add_argument_group self._positionals = add_group(_('positional arguments')) self._optionals = add_group(_('options')) self._subparsers = None # register types - def identity(string): - return string - self.register('type', None, identity) + self.register('type', None, _identity) # add help argument if necessary # (using explicit default to override global argument_default) @@ -1958,12 +2038,16 @@ def add_subparsers(self, **kwargs): self._subparsers = self._positionals # prog defaults to the usage message of this parser, skipping - # optional arguments and with no "usage:" prefix + # non-required optional arguments and with no "usage:" prefix if kwargs.get('prog') is None: - formatter = self._get_formatter() + # Create formatter without color to avoid storing ANSI codes in prog + formatter = self.formatter_class(prog=self.prog) + formatter._set_color(False) positionals = self._get_positional_actions() + required_optionals = [action for action in self._get_optional_actions() + if action.required] groups = self._mutually_exclusive_groups - formatter.add_usage(None, positionals, groups, '') + formatter.add_usage(None, required_optionals + positionals, groups, '') kwargs['prog'] = formatter.format_help().strip() # create the parsers action and add it to the positionals list @@ -2430,7 +2514,7 @@ def _parse_optional(self, arg_string): return None # if it doesn't start with a prefix, it was meant to be positional - if not arg_string[0] in self.prefix_chars: + if arg_string[0] not in self.prefix_chars: return None # if the option string is present in the parser, return the action @@ -2539,7 +2623,7 @@ def _get_nargs_pattern(self, action): # allow any number of options or arguments elif nargs == REMAINDER: - nargs_pattern = '([AO]*)' if option else '(.*)' + nargs_pattern = '(.*)' # allow one argument followed by any number of options or arguments elif nargs == PARSER: @@ -2692,14 +2776,16 @@ def _check_value(self, action, value): # Help-formatting methods # ======================= - def format_usage(self): - formatter = self._get_formatter() + def format_usage(self, formatter=None): + if formatter is None: + formatter = self._get_formatter() formatter.add_usage(self.usage, self._actions, self._mutually_exclusive_groups) return formatter.format_help() - def format_help(self): - formatter = self._get_formatter() + def format_help(self, formatter=None): + if formatter is None: + formatter = self._get_formatter() # usage formatter.add_usage(self.usage, self._actions, @@ -2721,11 +2807,18 @@ def format_help(self): # determine help from format above return formatter.format_help() - def _get_formatter(self): + def _get_formatter(self, file=None): formatter = self.formatter_class(prog=self.prog) - formatter._set_color(self.color) + formatter._set_color(self.color, file=file) return formatter + def _get_validation_formatter(self): + # Return cached formatter for read-only validation operations + # (_expand_help and _format_args). Avoids repeated slow _set_color calls. + if self._cached_formatter is None: + self._cached_formatter = self._get_formatter() + return self._cached_formatter + # ===================== # Help-printing methods # ===================== @@ -2733,12 +2826,26 @@ def _get_formatter(self): def print_usage(self, file=None): if file is None: file = _sys.stdout - self._print_message(self.format_usage(), file) + formatter = self._get_formatter(file=file) + try: + usage_text = self.format_usage(formatter=formatter) + except TypeError: + # Backward compatibility for formatter classes that + # do not accept the 'formatter' keyword argument. + usage_text = self.format_usage() + self._print_message(usage_text, file) def print_help(self, file=None): if file is None: file = _sys.stdout - self._print_message(self.format_help(), file) + formatter = self._get_formatter(file=file) + try: + help_text = self.format_help(formatter=formatter) + except TypeError: + # Backward compatibility for formatter classes that + # do not accept the 'formatter' keyword argument. + help_text = self.format_help() + self._print_message(help_text, file) def _print_message(self, message, file=None): if message: @@ -2748,6 +2855,14 @@ def _print_message(self, message, file=None): except (AttributeError, OSError): pass + def _get_theme(self, file=None): + from _colorize import can_colorize, get_theme + + if self.color and can_colorize(file=file): + return get_theme(force_color=True).argparse + else: + return get_theme(force_no_color=True).argparse + # =============== # Exiting methods # =============== @@ -2767,9 +2882,26 @@ def error(self, message): should either exit or raise an exception. """ self.print_usage(_sys.stderr) + theme = self._get_theme(file=_sys.stderr) + fmt = _('%(prog)s: error: %(message)s\n') + fmt = fmt.replace('error: %(message)s', + f'{theme.error}error:{theme.reset} {theme.message}%(message)s{theme.reset}') + args = {'prog': self.prog, 'message': message} - self.exit(2, _('%(prog)s: error: %(message)s\n') % args) + self.exit(2, fmt % args) def _warning(self, message): + theme = self._get_theme(file=_sys.stderr) + fmt = _('%(prog)s: warning: %(message)s\n') + fmt = fmt.replace('warning: %(message)s', + f'{theme.warning}warning:{theme.reset} {theme.message}%(message)s{theme.reset}') args = {'prog': self.prog, 'message': message} - self._print_message(_('%(prog)s: warning: %(message)s\n') % args, _sys.stderr) + self._print_message(fmt % args, _sys.stderr) + +def __getattr__(name): + if name == "__version__": + from warnings import _deprecated + + _deprecated("__version__", remove=(3, 20)) + return "1.1" # Do not change + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/ast.py b/Lib/ast.py index 983ac1710d0..ba4ee0197b8 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -21,10 +21,11 @@ :license: Python License. """ from _ast import * +lazy from _colorize import can_colorize, get_theme def parse(source, filename='', mode='exec', *, - type_comments=False, feature_version=None, optimize=-1): + type_comments=False, feature_version=None, optimize=-1, module=None): """ Parse the source into an AST node. Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). @@ -44,7 +45,8 @@ def parse(source, filename='', mode='exec', *, feature_version = minor # Else it should be an int giving the minor version for 3.x. return compile(source, filename, mode, flags, - _feature_version=feature_version, optimize=optimize) + _feature_version=feature_version, optimize=optimize, + module=module) def literal_eval(node_or_string): @@ -116,21 +118,32 @@ def _convert_literal(node): def dump( node, annotate_fields=True, include_attributes=False, *, - indent=None, show_empty=False, + color=False, indent=None, show_empty=False, ): """ Return a formatted dump of the tree in node. This is mainly useful for - debugging purposes. If annotate_fields is true (by default), - the returned string will show the names and the values for fields. - If annotate_fields is false, the result string will be more compact by - omitting unambiguous field names. Attributes such as line - numbers and column offsets are not dumped by default. If this is wanted, - include_attributes can be set to true. If indent is a non-negative - integer or string, then the tree will be pretty-printed with that indent - level. None (the default) selects the single line representation. + debugging purposes. + + If annotate_fields is true (by default), the returned string will show the + names and the values for fields. If annotate_fields is false, the result + string will be more compact by omitting unambiguous field names. + + Attributes such as line numbers and column offsets are not dumped by default. + If this is wanted, include_attributes can be set to true. + + If color is true, the returned string is syntax highlighted using ANSI + escape sequences. If color is false (the default), colored output is always + disabled. + + If indent is a non-negative integer or string, then the tree will be + pretty-printed with that indent level. If indent is None (the default), + the tree is dumped on a single line. + If show_empty is False, then empty lists and fields that are None will be omitted from the output for better readability. """ + t = get_theme(force_color=color, force_no_color=not color).ast + def _format(node, level=0): if indent is not None: level += 1 @@ -165,7 +178,9 @@ def _format(node, level=0): field_type = cls._field_types.get(name, object) if field_type is expr_context: if not keywords: - args_buffer.append(repr(value)) + args_buffer.append( + f'{t.node}{type(value).__name__}' + f'{t.reset}()') continue if not keywords: args.extend(args_buffer) @@ -173,7 +188,7 @@ def _format(node, level=0): value, simple = _format(value, level) allsimple = allsimple and simple if keywords: - args.append('%s=%s' % (name, value)) + args.append(f'{t.field}{name}{t.reset}={value}') else: args.append(value) if include_attributes and node._attributes: @@ -186,14 +201,21 @@ def _format(node, level=0): continue value, simple = _format(value, level) allsimple = allsimple and simple - args.append('%s=%s' % (name, value)) + args.append(f'{t.attribute}{name}{t.reset}={value}') + cls_name = f'{t.node}{cls.__name__}{t.reset}' if allsimple and len(args) <= 3: - return '%s(%s)' % (node.__class__.__name__, ', '.join(args)), not args - return '%s(%s%s)' % (node.__class__.__name__, prefix, sep.join(args)), False + return f'{cls_name}({", ".join(args)})', not args + return f'{cls_name}({prefix}{sep.join(args)})', False elif isinstance(node, list): if not node: return '[]', True return '[%s%s]' % (prefix, sep.join(_format(x, level)[0] for x in node)), False + if isinstance(node, bool) or node is None or node is Ellipsis: + return f'{t.keyword}{node!r}{t.reset}', True + if isinstance(node, (int, float, complex)): + return f'{t.number}{node!r}{t.reset}', True + if isinstance(node, (str, bytes)): + return f'{t.string}{node!r}{t.reset}', True return repr(node), True if not isinstance(node, AST): @@ -641,7 +663,7 @@ def main(args=None): import argparse import sys - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('infile', nargs='?', default='-', help='the file to parse; defaults to stdin') parser.add_argument('-m', '--mode', default='exec', @@ -660,7 +682,7 @@ def main(args=None): '(for example, 3.10)') parser.add_argument('-O', '--optimize', type=int, default=-1, metavar='LEVEL', - help='optimization level for parser (default -1)') + help='optimization level for parser') parser.add_argument('--show-empty', default=False, action='store_true', help='show empty lists and fields in dump output') args = parser.parse_args(args) @@ -687,6 +709,7 @@ def main(args=None): tree = parse(source, name, args.mode, type_comments=args.no_type_comments, feature_version=feature_version, optimize=args.optimize) print(dump(tree, include_attributes=args.include_attributes, + color=can_colorize(file=sys.stdout), indent=args.indent, show_empty=args.show_empty)) if __name__ == '__main__': diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index ff3a69d1e17..37eba9657ac 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -12,13 +12,16 @@ import types import warnings -from _colorize import get_theme -from _pyrepl.console import InteractiveColoredConsole +try: + from _colorize import get_theme + from _pyrepl.console import InteractiveColoredConsole as InteractiveConsole +except ModuleNotFoundError: + from code import InteractiveConsole from . import futures -class AsyncIOInteractiveConsole(InteractiveColoredConsole): +class AsyncIOInteractiveConsole(InteractiveConsole): def __init__(self, locals, loop): super().__init__(locals, filename="") @@ -74,7 +77,8 @@ def callback(): return except BaseException: if keyboard_interrupted: - self.write("\nKeyboardInterrupt\n") + if not CAN_USE_PYREPL: + self.write("\nKeyboardInterrupt\n") else: self.showtraceback() return self.STATEMENT_FAILED @@ -85,34 +89,43 @@ def run(self): global return_code try: - banner = ( - f'asyncio REPL {sys.version} on {sys.platform}\n' - f'Use "await" directly instead of "asyncio.run()".\n' - f'Type "help", "copyright", "credits" or "license" ' - f'for more information.\n' - ) + if not sys.flags.quiet: + banner = ( + f'asyncio REPL {sys.version} on {sys.platform}\n' + f'Use "await" directly instead of "asyncio.run()".\n' + f'Type "help", "copyright", "credits" or "license" ' + f'for more information.\n' + ) - console.write(banner) + console.write(banner) - if startup_path := os.getenv("PYTHONSTARTUP"): + if not sys.flags.isolated and (startup_path := os.getenv("PYTHONSTARTUP")): sys.audit("cpython.run_startup", startup_path) - - import tokenize - with tokenize.open(startup_path) as f: - startup_code = compile(f.read(), startup_path, "exec") + try: + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") exec(startup_code, console.locals) + except SystemExit: + raise + except BaseException: + console.showtraceback() ps1 = getattr(sys, "ps1", ">>> ") if CAN_USE_PYREPL: theme = get_theme().syntax ps1 = f"{theme.prompt}{ps1}{theme.reset}" - console.write(f"{ps1}import asyncio\n") + import_line = f'{theme.keyword}import{theme.reset} asyncio' + else: + import_line = "import asyncio" + console.write(f"{ps1}{import_line}\n") if CAN_USE_PYREPL: from _pyrepl.simple_interact import ( run_multiline_interactive_console, ) try: + sys.ps1 = ps1 run_multiline_interactive_console(console) except SystemExit: # expected via the `exit` and `quit` commands @@ -153,17 +166,29 @@ def interrupt(self) -> None: "ps", help="Display a table of all pending tasks in a process" ) ps.add_argument("pid", type=int, help="Process ID to inspect") + ps.add_argument( + "--retries", + type=int, + default=3, + help="Number of retries on transient attach errors", + ) pstree = subparsers.add_parser( "pstree", help="Display a tree of all pending tasks in a process" ) pstree.add_argument("pid", type=int, help="Process ID to inspect") + pstree.add_argument( + "--retries", + type=int, + default=3, + help="Number of retries on transient attach errors", + ) args = parser.parse_args() match args.command: case "ps": - asyncio.tools.display_awaited_by_tasks_table(args.pid) + asyncio.tools.display_awaited_by_tasks_table(args.pid, retries=args.retries) sys.exit(0) case "pstree": - asyncio.tools.display_awaited_by_tasks_tree(args.pid) + asyncio.tools.display_awaited_by_tasks_tree(args.pid, retries=args.retries) sys.exit(0) case None: pass # continue to the interactive shell @@ -179,7 +204,10 @@ def interrupt(self) -> None: if os.getenv('PYTHON_BASIC_REPL'): CAN_USE_PYREPL = False else: - from _pyrepl.main import CAN_USE_PYREPL + try: + from _pyrepl.main import CAN_USE_PYREPL + except ModuleNotFoundError: + CAN_USE_PYREPL = False return_code = 0 loop = asyncio.new_event_loop() @@ -235,4 +263,5 @@ def interrupt(self) -> None: break console.write('exiting asyncio REPL...\n') + loop.close() sys.exit(return_code) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 8cbb71f7085..7a6837546d9 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -14,19 +14,21 @@ """ import collections +import contextvars import collections.abc import concurrent.futures import errno import heapq import itertools +import math import os import socket import stat import subprocess +import sys import threading import time import traceback -import sys import warnings import weakref @@ -289,6 +291,7 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, self._ssl_shutdown_timeout = ssl_shutdown_timeout self._serving = False self._serving_forever_fut = None + self._context = contextvars.copy_context() def __repr__(self): return f'<{self.__class__.__name__} sockets={self.sockets!r}>' @@ -318,7 +321,7 @@ def _start_serving(self): self._loop._start_serving( self._protocol_factory, sock, self._ssl_context, self, self._backlog, self._ssl_handshake_timeout, - self._ssl_shutdown_timeout) + self._ssl_shutdown_timeout, context=self._context) def get_loop(self): return self._loop @@ -380,6 +383,7 @@ async def serve_forever(self): except exceptions.CancelledError: try: self.close() + self.close_clients() await self.wait_closed() finally: raise @@ -507,7 +511,8 @@ def _make_ssl_transport( extra=None, server=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, - call_connection_made=True): + call_connection_made=True, + context=None): """Create SSL transport.""" raise NotImplementedError @@ -949,7 +954,7 @@ async def sock_sendfile(self, sock, file, offset=0, count=None, try: return await self._sock_sendfile_native(sock, file, offset, count) - except exceptions.SendfileNotAvailableError as exc: + except exceptions.SendfileNotAvailableError: if not fallback: raise return await self._sock_sendfile_fallback(sock, file, @@ -1211,9 +1216,10 @@ async def _create_connection_transport( self, sock, protocol_factory, ssl, server_hostname, server_side=False, ssl_handshake_timeout=None, - ssl_shutdown_timeout=None): + ssl_shutdown_timeout=None, context=None): sock.setblocking(False) + context = context if context is not None else contextvars.copy_context() protocol = protocol_factory() waiter = self.create_future() @@ -1223,9 +1229,10 @@ async def _create_connection_transport( sock, protocol, sslcontext, waiter, server_side=server_side, server_hostname=server_hostname, ssl_handshake_timeout=ssl_handshake_timeout, - ssl_shutdown_timeout=ssl_shutdown_timeout) + ssl_shutdown_timeout=ssl_shutdown_timeout, + context=context) else: - transport = self._make_socket_transport(sock, protocol, waiter) + transport = self._make_socket_transport(sock, protocol, waiter, context=context) try: await waiter @@ -1270,7 +1277,7 @@ async def sendfile(self, transport, file, offset=0, count=None, try: return await self._sendfile_native(transport, file, offset, count) - except exceptions.SendfileNotAvailableError as exc: + except exceptions.SendfileNotAvailableError: if not fallback: raise @@ -1345,6 +1352,17 @@ async def start_tls(self, transport, protocol, sslcontext, *, # have a chance to get called before "ssl_protocol.connection_made()". transport.pause_reading() + # gh-142352: move buffered StreamReader data to SSLProtocol + if server_side: + from .streams import StreamReaderProtocol + if isinstance(protocol, StreamReaderProtocol): + stream_reader = getattr(protocol, '_stream_reader', None) + if stream_reader is not None: + buffer = stream_reader._buffer + if buffer: + ssl_protocol._incoming.write(buffer) + buffer.clear() + transport.set_protocol(ssl_protocol) conmade_cb = self.call_soon(ssl_protocol.connection_made, transport) resume_cb = self.call_soon(transport.resume_reading) @@ -2011,7 +2029,10 @@ def _run_once(self): event_list = None # Handle 'later' callbacks that are ready. - end_time = self.time() + self._clock_resolution + now = self.time() + # Ensure that `end_time` is strictly increasing + # when the clock resolution is too small. + end_time = now + max(self._clock_resolution, math.ulp(now)) while self._scheduled: handle = self._scheduled[0] if handle._when >= end_time: diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index d40af422e61..224b1883808 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -26,6 +26,7 @@ def __init__(self, loop, protocol, args, shell, self._pending_calls = collections.deque() self._pipes = {} self._finished = False + self._pipes_connected = False if stdin == subprocess.PIPE: self._pipes[0] = None @@ -213,6 +214,7 @@ async def _connect_pipes(self, waiter): else: if waiter is not None and not waiter.cancelled(): waiter.set_result(None) + self._pipes_connected = True def _call(self, cb, *data): if self._pending_calls is not None: @@ -256,6 +258,15 @@ def _try_finish(self): assert not self._finished if self._returncode is None: return + if not self._pipes_connected: + # self._pipes_connected can be False if not all pipes were connected + # because either the process failed to start or the self._connect_pipes task + # got cancelled. In this broken state we consider all pipes disconnected and + # to avoid hanging forever in self._wait as otherwise _exit_waiters + # would never be woken up, we wake them up here. + for waiter in self._exit_waiters: + if not waiter.done(): + waiter.set_result(self._returncode) if all(p is not None and p.disconnected for p in self._pipes.values()): self._finished = True @@ -267,7 +278,7 @@ def _call_connection_lost(self, exc): finally: # wake up futures waiting for wait() for waiter in self._exit_waiters: - if not waiter.cancelled(): + if not waiter.done(): waiter.set_result(self._returncode) self._exit_waiters = None self._loop = None diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 6bd00a64478..11858a0274a 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -79,6 +79,10 @@ def __init__(self, *, loop=None): loop object used by the future. If it's not provided, the future uses the default event loop. """ + if self._loop is not None: + raise RuntimeError(f"{self.__class__.__name__} object is already " + "initialized") + if loop is None: self._loop = events.get_event_loop() else: @@ -389,7 +393,7 @@ def _set_state(future, other): def _call_check_cancel(destination): if destination.cancelled(): - if source_loop is None or source_loop is dest_loop: + if source_loop is None or source_loop is events._get_running_loop(): source.cancel() else: source_loop.call_soon_threadsafe(source.cancel) @@ -398,7 +402,7 @@ def _call_set_state(source): if (destination.cancelled() and dest_loop is not None and dest_loop.is_closed()): return - if dest_loop is None or dest_loop is source_loop: + if dest_loop is None or dest_loop is events._get_running_loop(): _set_state(destination, source) else: if dest_loop.is_closed(): diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index f404273c3ae..2dc1569d780 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -642,7 +642,7 @@ def __init__(self, proactor): signal.set_wakeup_fd(self._csock.fileno()) def _make_socket_transport(self, sock, protocol, waiter=None, - extra=None, server=None): + extra=None, server=None, context=None): return _ProactorSocketTransport(self, sock, protocol, waiter, extra, server) @@ -651,7 +651,7 @@ def _make_ssl_transport( *, server_side=False, server_hostname=None, extra=None, server=None, ssl_handshake_timeout=None, - ssl_shutdown_timeout=None): + ssl_shutdown_timeout=None, context=None): ssl_protocol = sslproto.SSLProtocol( self, protocol, sslcontext, waiter, server_side, server_hostname, @@ -733,7 +733,7 @@ async def sock_accept(self, sock): async def _sock_sendfile_native(self, sock, file, offset, count): try: fileno = file.fileno() - except (AttributeError, io.UnsupportedOperation) as err: + except (AttributeError, io.UnsupportedOperation): raise exceptions.SendfileNotAvailableError("not a regular file") try: fsize = os.fstat(fileno).st_size @@ -837,7 +837,7 @@ def _write_to_self(self): def _start_serving(self, protocol_factory, sock, sslcontext=None, server=None, backlog=100, ssl_handshake_timeout=None, - ssl_shutdown_timeout=None): + ssl_shutdown_timeout=None, context=None): def loop(f=None): try: diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 084fccaaff2..756216fac80 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -37,7 +37,7 @@ class Queue(mixins._LoopBoundMixin): is an integer greater than 0, then "await put()" will block when the queue reaches maxsize, until an item is removed by get(). - Unlike the standard library Queue, you can reliably know this Queue's size + Unlike queue.Queue, you can reliably know this Queue's size with qsize(), since your single-threaded asyncio application won't be interrupted between calling qsize() and doing an operation on the Queue. """ diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index a7e27ccf0aa..961dbfb4b96 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -67,10 +67,10 @@ def __init__(self, selector=None): self._transports = weakref.WeakValueDictionary() def _make_socket_transport(self, sock, protocol, waiter=None, *, - extra=None, server=None): + extra=None, server=None, context=None): self._ensure_fd_no_transport(sock) return _SelectorSocketTransport(self, sock, protocol, waiter, - extra, server) + extra, server, context=context) def _make_ssl_transport( self, rawsock, protocol, sslcontext, waiter=None, @@ -78,16 +78,17 @@ def _make_ssl_transport( extra=None, server=None, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, + context=None, ): self._ensure_fd_no_transport(rawsock) ssl_protocol = sslproto.SSLProtocol( self, protocol, sslcontext, waiter, server_side, server_hostname, ssl_handshake_timeout=ssl_handshake_timeout, - ssl_shutdown_timeout=ssl_shutdown_timeout + ssl_shutdown_timeout=ssl_shutdown_timeout, ) _SelectorSocketTransport(self, rawsock, ssl_protocol, - extra=extra, server=server) + extra=extra, server=server, context=context) return ssl_protocol._app_transport def _make_datagram_transport(self, sock, protocol, @@ -159,16 +160,16 @@ def _write_to_self(self): def _start_serving(self, protocol_factory, sock, sslcontext=None, server=None, backlog=100, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, - ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT): + ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, context=None): self._add_reader(sock.fileno(), self._accept_connection, protocol_factory, sock, sslcontext, server, backlog, - ssl_handshake_timeout, ssl_shutdown_timeout) + ssl_handshake_timeout, ssl_shutdown_timeout, context) def _accept_connection( self, protocol_factory, sock, sslcontext=None, server=None, backlog=100, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, - ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT): + ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, context=None): # This method is only called once for each event loop tick where the # listening socket has triggered an EVENT_READ. There may be multiple # connections waiting for an .accept() so it is called in a loop. @@ -204,21 +205,22 @@ def _accept_connection( self._start_serving, protocol_factory, sock, sslcontext, server, backlog, ssl_handshake_timeout, - ssl_shutdown_timeout) + ssl_shutdown_timeout, context) else: raise # The event loop will catch, log and ignore it. else: extra = {'peername': addr} + conn_context = context.copy() if context is not None else None accept = self._accept_connection2( protocol_factory, conn, extra, sslcontext, server, - ssl_handshake_timeout, ssl_shutdown_timeout) - self.create_task(accept) + ssl_handshake_timeout, ssl_shutdown_timeout, context=conn_context) + self.create_task(accept, context=conn_context) async def _accept_connection2( self, protocol_factory, conn, extra, sslcontext=None, server=None, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, - ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT): + ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, context=None): protocol = None transport = None try: @@ -229,11 +231,12 @@ async def _accept_connection2( conn, protocol, sslcontext, waiter=waiter, server_side=True, extra=extra, server=server, ssl_handshake_timeout=ssl_handshake_timeout, - ssl_shutdown_timeout=ssl_shutdown_timeout) + ssl_shutdown_timeout=ssl_shutdown_timeout, + context=context) else: transport = self._make_socket_transport( conn, protocol, waiter=waiter, extra=extra, - server=server) + server=server, context=context) try: await waiter @@ -275,9 +278,9 @@ def _ensure_fd_no_transport(self, fd): f'File descriptor {fd!r} is used by transport ' f'{transport!r}') - def _add_reader(self, fd, callback, *args): + def _add_reader(self, fd, callback, *args, context=None): self._check_closed() - handle = events.Handle(callback, args, self, None) + handle = events.Handle(callback, args, self, context=context) key = self._selector.get_map().get(fd) if key is None: self._selector.register(fd, selectors.EVENT_READ, @@ -309,9 +312,9 @@ def _remove_reader(self, fd): else: return False - def _add_writer(self, fd, callback, *args): + def _add_writer(self, fd, callback, *args, context=None): self._check_closed() - handle = events.Handle(callback, args, self, None) + handle = events.Handle(callback, args, self, context=context) key = self._selector.get_map().get(fd) if key is None: self._selector.register(fd, selectors.EVENT_WRITE, @@ -770,7 +773,7 @@ class _SelectorTransport(transports._FlowControlMixin, # exception) _sock = None - def __init__(self, loop, sock, protocol, extra=None, server=None): + def __init__(self, loop, sock, protocol, extra=None, server=None, context=None): super().__init__(extra, loop) self._extra['socket'] = trsock.TransportSocket(sock) try: @@ -784,12 +787,13 @@ def __init__(self, loop, sock, protocol, extra=None, server=None): self._extra['peername'] = None self._sock = sock self._sock_fd = sock.fileno() - + self._context = context self._protocol_connected = False self.set_protocol(protocol) self._server = server self._buffer = collections.deque() + self._buffer_size = 0 self._conn_lost = 0 # Set when call to connection_lost scheduled. self._closing = False # Set when close() called. self._paused = False # Set when pause_reading() called @@ -866,7 +870,7 @@ def close(self): if not self._buffer: self._conn_lost += 1 self._loop._remove_writer(self._sock_fd) - self._loop.call_soon(self._call_connection_lost, None) + self._call_soon(self._call_connection_lost, None) def __del__(self, _warn=warnings.warn): if self._sock is not None: @@ -894,12 +898,13 @@ def _force_close(self, exc): return if self._buffer: self._buffer.clear() + self._buffer_size = 0 self._loop._remove_writer(self._sock_fd) if not self._closing: self._closing = True self._loop._remove_reader(self._sock_fd) self._conn_lost += 1 - self._loop.call_soon(self._call_connection_lost, exc) + self._call_soon(self._call_connection_lost, exc) def _call_connection_lost(self, exc): try: @@ -916,13 +921,18 @@ def _call_connection_lost(self, exc): self._server = None def get_write_buffer_size(self): - return sum(map(len, self._buffer)) + return self._buffer_size def _add_reader(self, fd, callback, *args): if not self.is_reading(): return - self._loop._add_reader(fd, callback, *args) + self._loop._add_reader(fd, callback, *args, context=self._context) + def _add_writer(self, fd, callback, *args): + self._loop._add_writer(fd, callback, *args, context=self._context) + + def _call_soon(self, callback, *args): + self._loop.call_soon(callback, *args, context=self._context) class _SelectorSocketTransport(_SelectorTransport): @@ -930,10 +940,9 @@ class _SelectorSocketTransport(_SelectorTransport): _sendfile_compatible = constants._SendfileMode.TRY_NATIVE def __init__(self, loop, sock, protocol, waiter=None, - extra=None, server=None): - + extra=None, server=None, context=None): self._read_ready_cb = None - super().__init__(loop, sock, protocol, extra, server) + super().__init__(loop, sock, protocol, extra, server, context) self._eof = False self._empty_waiter = None if _HAS_SENDMSG: @@ -945,14 +954,12 @@ def __init__(self, loop, sock, protocol, waiter=None, # decreases the latency (in some cases significantly.) base_events._set_nodelay(self._sock) - self._loop.call_soon(self._protocol.connection_made, self) + self._call_soon(self._protocol.connection_made, self) # only start reading when connection_made() has been called - self._loop.call_soon(self._add_reader, - self._sock_fd, self._read_ready) + self._call_soon(self._add_reader, self._sock_fd, self._read_ready) if waiter is not None: # only wake up the waiter when connection_made() has been called - self._loop.call_soon(futures._set_result_unless_cancelled, - waiter, None) + self._call_soon(futures._set_result_unless_cancelled, waiter, None) def set_protocol(self, protocol): if isinstance(protocol, protocols.BufferedProtocol): @@ -1050,8 +1057,8 @@ def _read_ready__on_eof(self): def write(self, data): if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError(f'data argument must be a bytes-like object, ' - f'not {type(data).__name__!r}') + raise TypeError(f'data argument must be a bytes, bytearray, or memoryview ' + f'object, not {type(data).__name__!r}') if self._eof: raise RuntimeError('Cannot call write() after write_eof()') if self._empty_waiter is not None: @@ -1081,10 +1088,11 @@ def write(self, data): if not data: return # Not all was written; register write handler. - self._loop._add_writer(self._sock_fd, self._write_ready) + self._add_writer(self._sock_fd, self._write_ready) # Add it to the buffer. self._buffer.append(data) + self._buffer_size += len(data) self._maybe_pause_protocol() def _get_sendmsg_buffer(self): @@ -1104,6 +1112,7 @@ def _write_sendmsg(self): except BaseException as exc: self._loop._remove_writer(self._sock_fd) self._buffer.clear() + self._buffer_size = 0 self._fatal_error(exc, 'Fatal write error on socket transport') if self._empty_waiter is not None: self._empty_waiter.set_exception(exc) @@ -1119,6 +1128,7 @@ def _write_sendmsg(self): self._sock.shutdown(socket.SHUT_WR) def _adjust_leftover_buffer(self, nbytes: int) -> None: + self._buffer_size -= nbytes buffer = self._buffer while nbytes: b = buffer.popleft() @@ -1139,13 +1149,16 @@ def _write_send(self): if n != len(buffer): # Not all data was written self._buffer.appendleft(buffer[n:]) + self._buffer_size -= n except (BlockingIOError, InterruptedError): - pass + self._buffer.appendleft(buffer) + return except (SystemExit, KeyboardInterrupt): raise except BaseException as exc: self._loop._remove_writer(self._sock_fd) self._buffer.clear() + self._buffer_size = 0 self._fatal_error(exc, 'Fatal write error on socket transport') if self._empty_waiter is not None: self._empty_waiter.set_exception(exc) @@ -1181,11 +1194,13 @@ def writelines(self, list_of_data): self._conn_lost += 1 return - self._buffer.extend([memoryview(data) for data in list_of_data]) + for data in list_of_data: + self._buffer.append(memoryview(data)) + self._buffer_size += len(data) self._write_ready() # If the entire buffer couldn't be written, register a write handler if self._buffer: - self._loop._add_writer(self._sock_fd, self._write_ready) + self._add_writer(self._sock_fd, self._write_ready) self._maybe_pause_protocol() def can_write_eof(self): @@ -1226,14 +1241,12 @@ def __init__(self, loop, sock, protocol, address=None, super().__init__(loop, sock, protocol, extra) self._address = address self._buffer_size = 0 - self._loop.call_soon(self._protocol.connection_made, self) + self._call_soon(self._protocol.connection_made, self) # only start reading when connection_made() has been called - self._loop.call_soon(self._add_reader, - self._sock_fd, self._read_ready) + self._call_soon(self._add_reader, self._sock_fd, self._read_ready) if waiter is not None: # only wake up the waiter when connection_made() has been called - self._loop.call_soon(futures._set_result_unless_cancelled, - waiter, None) + self._call_soon(futures._set_result_unless_cancelled, waiter, None) def get_write_buffer_size(self): return self._buffer_size @@ -1280,7 +1293,7 @@ def sendto(self, data, addr=None): self._sock.sendto(data, addr) return except (BlockingIOError, InterruptedError): - self._loop._add_writer(self._sock_fd, self._sendto_ready) + self._add_writer(self._sock_fd, self._sendto_ready) except OSError as exc: self._protocol.error_received(exc) return diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index c8c01f36474..d2db1a930c2 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -214,7 +214,6 @@ def _stream_reader(self): return self._stream_reader_wr() def _replace_transport(self, transport): - loop = self._loop self._transport = transport self._over_ssl = transport.get_extra_info('sslcontext') is not None @@ -668,8 +667,7 @@ async def readuntil(self, separator=b'\n'): # adds data which makes separator be found. That's why we check for # EOF *after* inspecting the buffer. if self._eof: - chunk = bytes(self._buffer) - self._buffer.clear() + chunk = self._buffer.take_bytes() raise exceptions.IncompleteReadError(chunk, None) # _wait_for_data() will resume reading if stream was paused. @@ -679,10 +677,9 @@ async def readuntil(self, separator=b'\n'): raise exceptions.LimitOverrunError( 'Separator is found, but chunk is longer than limit', match_start) - chunk = self._buffer[:match_end] - del self._buffer[:match_end] + chunk = self._buffer.take_bytes(match_end) self._maybe_resume_transport() - return bytes(chunk) + return chunk async def read(self, n=-1): """Read up to `n` bytes from the stream. @@ -717,20 +714,16 @@ async def read(self, n=-1): # collect everything in self._buffer, but that would # deadlock if the subprocess sends more than self.limit # bytes. So just call self.read(self._limit) until EOF. - blocks = [] - while True: - block = await self.read(self._limit) - if not block: - break - blocks.append(block) - return b''.join(blocks) + joined = bytearray() + while block := await self.read(self._limit): + joined += block + return joined.take_bytes() if not self._buffer and not self._eof: await self._wait_for_data('read') # This will work right even if buffer is less than n bytes - data = bytes(memoryview(self._buffer)[:n]) - del self._buffer[:n] + data = self._buffer.take_bytes(min(len(self._buffer), n)) self._maybe_resume_transport() return data @@ -761,18 +754,12 @@ async def readexactly(self, n): while len(self._buffer) < n: if self._eof: - incomplete = bytes(self._buffer) - self._buffer.clear() + incomplete = self._buffer.take_bytes() raise exceptions.IncompleteReadError(incomplete, n) await self._wait_for_data('readexactly') - if len(self._buffer) == n: - data = bytes(self._buffer) - self._buffer.clear() - else: - data = bytes(memoryview(self._buffer)[:n]) - del self._buffer[:n] + data = self._buffer.take_bytes(n) self._maybe_resume_transport() return data diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 00e8f6d5d1a..45dfebc6590 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -37,6 +37,7 @@ def __init__(self): self._errors = [] self._base_error = None self._on_completed_fut = None + self._cancel_on_enter = False def __repr__(self): info = [''] @@ -63,6 +64,8 @@ async def __aenter__(self): raise RuntimeError( f'TaskGroup {self!r} cannot determine the parent task') self._entered = True + if self._cancel_on_enter: + self.cancel() return self @@ -178,6 +181,9 @@ async def _aexit(self, et, exc): finally: exc = None + # Suppress any remaining exception (exceptions deserving to be raised + # were raised above). + return True def create_task(self, coro, **kwargs): """Create a new task in this group and return it. @@ -278,3 +284,30 @@ def _on_task_done(self, task): self._abort() self._parent_cancel_requested = True self._parent_task.cancel() + + def cancel(self): + """Cancel the task group + + `cancel()` will be called on any tasks in the group that aren't yet + done, as well as the parent (body) of the group. This will cause the + task group context manager to exit *without* `asyncio.CancelledError` + being raised. + + If `cancel()` is called before entering the task group, the group will be + cancelled upon entry. This is useful for patterns where one piece of + code passes an unused TaskGroup instance to another in order to have + the ability to cancel anything run within the group. + + `cancel()` is idempotent and may be called after the task group has + already exited. + """ + if not self._entered: + self._cancel_on_enter = True + return + if self._exiting and not self._tasks: + return + if not self._aborting: + self._abort() + if self._parent_task and not self._parent_cancel_requested: + self._parent_cancel_requested = True + self._parent_task.cancel() diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 2683f34cc71..2ac1738d15c 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -1,6 +1,6 @@ """Tools to analyze tasks running in asyncio programs.""" -from collections import defaultdict, namedtuple +from collections import defaultdict from itertools import count from enum import Enum import sys @@ -27,10 +27,10 @@ def __init__( # ─── indexing helpers ─────────────────────────────────────────── def _format_stack_entry(elem: str|FrameInfo) -> str: if not isinstance(elem, str): - if elem.lineno == 0 and elem.filename == "": + if elem.location.lineno == 0 and elem.filename == "": return f"{elem.funcname}" else: - return f"{elem.funcname} {elem.filename}:{elem.lineno}" + return f"{elem.funcname} {elem.filename}:{elem.location.lineno}" return elem @@ -222,20 +222,47 @@ def _print_cycle_exception(exception: CycleFoundException): print(f"cycle: {inames}", file=sys.stderr) -def _get_awaited_by_tasks(pid: int) -> list: - try: - return get_all_awaited_by(pid) - except RuntimeError as e: - while e.__context__ is not None: - e = e.__context__ - print(f"Error retrieving tasks: {e}") - sys.exit(1) +def exit_with_permission_help_text(): + """ + Prints a message pointing to platform-specific permission help text and exits the program. + This function is called when a PermissionError is encountered while trying + to attach to a process. + """ + print( + "Error: The specified process cannot be attached to due to insufficient permissions.\n" + "See the Python documentation for details on required privileges and troubleshooting:\n" + "https://docs.python.org/3/howto/remote_debugging.html#permission-requirements\n", + file=sys.stderr, + ) + sys.exit(1) -def display_awaited_by_tasks_table(pid: int) -> None: +_TRANSIENT_ERRORS = (RuntimeError, OSError, UnicodeDecodeError, MemoryError) + + +def _get_awaited_by_tasks(pid: int, retries: int = 3) -> list: + for attempt in range(retries + 1): + try: + return get_all_awaited_by(pid) + except PermissionError: + exit_with_permission_help_text() + except ProcessLookupError: + print(f"Error: process {pid} not found.", file=sys.stderr) + sys.exit(1) + except _TRANSIENT_ERRORS as e: + if attempt < retries: + continue + if isinstance(e, RuntimeError): + while e.__context__ is not None: + e = e.__context__ + print(f"Error retrieving tasks: {e}", file=sys.stderr) + sys.exit(1) + + +def display_awaited_by_tasks_table(pid: int, retries: int = 3) -> None: """Build and print a table of all pending tasks under `pid`.""" - tasks = _get_awaited_by_tasks(pid) + tasks = _get_awaited_by_tasks(pid, retries=retries) table = build_task_table(tasks) # Print the table in a simple tabular format print( @@ -246,10 +273,10 @@ def display_awaited_by_tasks_table(pid: int) -> None: print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<50} {row[5]:<15} {row[6]:<15}") -def display_awaited_by_tasks_tree(pid: int) -> None: +def display_awaited_by_tasks_tree(pid: int, retries: int = 3) -> None: """Build and print a tree of all pending tasks under `pid`.""" - tasks = _get_awaited_by_tasks(pid) + tasks = _get_awaited_by_tasks(pid, retries=retries) try: result = build_async_tree(tasks) except CycleFoundException as e: diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 1c1458127db..49e8067ee7b 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -359,7 +359,7 @@ async def _sock_sendfile_native(self, sock, file, offset, count): "os.sendfile() is not available") try: fileno = file.fileno() - except (AttributeError, io.UnsupportedOperation) as err: + except (AttributeError, io.UnsupportedOperation): raise exceptions.SendfileNotAvailableError("not a regular file") try: fsize = os.fstat(fileno).st_size diff --git a/Lib/asyncio/windows_utils.py b/Lib/asyncio/windows_utils.py index ef277fac3e2..acd49441131 100644 --- a/Lib/asyncio/windows_utils.py +++ b/Lib/asyncio/windows_utils.py @@ -10,7 +10,6 @@ import msvcrt import os import subprocess -import tempfile import warnings @@ -24,6 +23,7 @@ PIPE = subprocess.PIPE STDOUT = subprocess.STDOUT _mmap_counter = itertools.count() +_MAX_PIPE_ATTEMPTS = 20 # Replacement for os.pipe() using handles instead of fds @@ -31,10 +31,6 @@ def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE): """Like os.pipe() but with overlapped support and using handles not fds.""" - address = tempfile.mktemp( - prefix=r'\\.\pipe\python-pipe-{:d}-{:d}-'.format( - os.getpid(), next(_mmap_counter))) - if duplex: openmode = _winapi.PIPE_ACCESS_DUPLEX access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE @@ -56,9 +52,20 @@ def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE): h1 = h2 = None try: - h1 = _winapi.CreateNamedPipe( - address, openmode, _winapi.PIPE_WAIT, - 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL) + for attempts in itertools.count(): + address = r'\\.\pipe\python-pipe-{:d}-{:d}-{}'.format( + os.getpid(), next(_mmap_counter), os.urandom(8).hex()) + try: + h1 = _winapi.CreateNamedPipe( + address, openmode, _winapi.PIPE_WAIT, + 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL) + break + except OSError as e: + if attempts >= _MAX_PIPE_ATTEMPTS: + raise + if e.winerror not in (_winapi.ERROR_PIPE_BUSY, + _winapi.ERROR_ACCESS_DENIED): + raise h2 = _winapi.CreateFile( address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING, diff --git a/Lib/base64.py b/Lib/base64.py index 5d78cc09f40..4b810e08569 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -4,7 +4,6 @@ # Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support # Modified 22-May-2007 by Guido van Rossum to use bytes everywhere -import struct import binascii @@ -26,6 +25,8 @@ ] +_NOT_SPECIFIED = ['NOT SPECIFIED'] + bytes_types = (bytes, bytearray) # Types acceptable as binary data def _bytes_from_decode_data(s): @@ -45,44 +46,90 @@ def _bytes_from_decode_data(s): # Base64 encoding/decoding uses binascii -def b64encode(s, altchars=None): +def b64encode(s, altchars=None, *, padded=True, wrapcol=0): """Encode the bytes-like object s using Base64 and return a bytes object. Optional altchars should be a byte string of length 2 which specifies an alternative alphabet for the '+' and '/' characters. This allows an application to e.g. generate url or filesystem safe Base64 strings. + + If padded is false, omit padding in the output. + + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. """ - encoded = binascii.b2a_base64(s, newline=False) if altchars is not None: - assert len(altchars) == 2, repr(altchars) - return encoded.translate(bytes.maketrans(b'+/', altchars)) - return encoded + if len(altchars) != 2: + raise ValueError(f'invalid altchars: {altchars!r}') + alphabet = binascii.BASE64_ALPHABET[:-2] + altchars + return binascii.b2a_base64(s, padded=padded, wrapcol=wrapcol, newline=False, + alphabet=alphabet) + return binascii.b2a_base64(s, padded=padded, wrapcol=wrapcol, newline=False) -def b64decode(s, altchars=None, validate=False): +def b64decode(s, altchars=None, validate=_NOT_SPECIFIED, + *, padded=True, ignorechars=_NOT_SPECIFIED, canonical=False): """Decode the Base64 encoded bytes-like object or ASCII string s. Optional altchars must be a bytes-like object or ASCII string of length 2 which specifies the alternative alphabet used instead of the '+' and '/' characters. + If padded is false, padding in input is not required. + The result is returned as a bytes object. A binascii.Error is raised if s is incorrectly padded. - If validate is False (the default), characters that are neither in the - normal base-64 alphabet nor the alternative alphabet are discarded prior - to the padding check. If validate is True, these non-alphabet characters - in the input result in a binascii.Error. + If ignorechars is specified, it should be a byte string containing + characters to ignore from the input. The default value of validate is + True if ignorechars is specified, False otherwise. + + If validate is false, characters that are neither in the normal base-64 + alphabet nor the alternative alphabet are discarded prior to the + padding check. If validate is true, these non-alphabet characters in + the input result in a binascii.Error if they are not in ignorechars. For more information about the strict base64 check, see: https://docs.python.org/3.11/library/binascii.html#binascii.a2b_base64 """ s = _bytes_from_decode_data(s) + if validate is _NOT_SPECIFIED: + validate = ignorechars is not _NOT_SPECIFIED + badchar = None if altchars is not None: altchars = _bytes_from_decode_data(altchars) - assert len(altchars) == 2, repr(altchars) - s = s.translate(bytes.maketrans(altchars, b'+/')) - return binascii.a2b_base64(s, strict_mode=validate) + if len(altchars) != 2: + raise ValueError(f'invalid altchars: {altchars!r}') + if ignorechars is _NOT_SPECIFIED: + for b in b'+/': + if b not in altchars and b in s: + badchar = b + break + s = s.translate(bytes.maketrans(altchars, b'+/')) + else: + alphabet = binascii.BASE64_ALPHABET[:-2] + altchars + return binascii.a2b_base64(s, strict_mode=validate, + alphabet=alphabet, + padded=padded, ignorechars=ignorechars, + canonical=canonical) + if ignorechars is _NOT_SPECIFIED: + ignorechars = b'' + result = binascii.a2b_base64(s, strict_mode=validate, + padded=padded, ignorechars=ignorechars, + canonical=canonical) + if badchar is not None: + import warnings + if validate: + warnings.warn(f'invalid character {chr(badchar)!a} in Base64 data ' + f'with altchars={altchars!r} and validate=True ' + f'will be an error in future Python versions', + DeprecationWarning, stacklevel=2) + else: + warnings.warn(f'invalid character {chr(badchar)!a} in Base64 data ' + f'with altchars={altchars!r} and validate=False ' + f'will be discarded in future Python versions', + FutureWarning, stacklevel=2) + return result def standard_b64encode(s): @@ -103,19 +150,21 @@ def standard_b64decode(s): return b64decode(s) -_urlsafe_encode_translation = bytes.maketrans(b'+/', b'-_') _urlsafe_decode_translation = bytes.maketrans(b'-_', b'+/') -def urlsafe_b64encode(s): +def urlsafe_b64encode(s, *, padded=True): """Encode bytes using the URL- and filesystem-safe Base64 alphabet. Argument s is a bytes-like object to encode. The result is returned as a bytes object. The alphabet uses '-' instead of '+' and '_' instead of '/'. - """ - return b64encode(s).translate(_urlsafe_encode_translation) -def urlsafe_b64decode(s): + If padded is false, omit padding in the output. + """ + return binascii.b2a_base64(s, padded=padded, newline=False, + alphabet=binascii.URLSAFE_BASE64_ALPHABET) + +def urlsafe_b64decode(s, *, padded=False): """Decode bytes using the URL- and filesystem-safe Base64 alphabet. Argument s is a bytes-like object or ASCII string to decode. The result @@ -124,23 +173,46 @@ def urlsafe_b64decode(s): alphabet, and are not a plus '+' or slash '/', are discarded prior to the padding check. + If padded is false, padding in input is not required. + The alphabet uses '-' instead of '+' and '_' instead of '/'. """ s = _bytes_from_decode_data(s) + badchar = None + for b in b'+/': + if b in s: + badchar = b + break s = s.translate(_urlsafe_decode_translation) - return b64decode(s) + result = binascii.a2b_base64(s, strict_mode=False, padded=padded) + if badchar is not None: + import warnings + warnings.warn(f'invalid character {chr(badchar)!a} in URL-safe Base64 data ' + f'will be discarded in future Python versions', + FutureWarning, stacklevel=2) + return result # Base32 encoding/decoding must be done in Python _B32_ENCODE_DOCSTRING = ''' Encode the bytes-like objects using {encoding} and return a bytes object. + +If padded is false, omit padding in the output. + +If wrapcol is non-zero, insert a newline (b'\\n') character after at most +every wrapcol characters. ''' _B32_DECODE_DOCSTRING = ''' Decode the {encoding} encoded bytes-like object or ASCII string s. Optional casefold is a flag specifying whether a lowercase alphabet is acceptable as input. For security purposes, the default is False. + +If padded is false, padding in input is not required. + +ignorechars should be a byte string containing characters to ignore +from the input. {extra_args} The result is returned as a bytes object. A binascii.Error is raised if the input is incorrectly padded or if there are non-alphabet @@ -155,108 +227,41 @@ def urlsafe_b64decode(s): the letter O). For security purposes the default is None, so that 0 and 1 are not allowed in the input. ''' -_b32alphabet = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567' -_b32hexalphabet = b'0123456789ABCDEFGHIJKLMNOPQRSTUV' -_b32tab2 = {} -_b32rev = {} -def _b32encode(alphabet, s): - # Delay the initialization of the table to not waste memory - # if the function is never called - if alphabet not in _b32tab2: - b32tab = [bytes((i,)) for i in alphabet] - _b32tab2[alphabet] = [a + b for a in b32tab for b in b32tab] - b32tab = None +def b32encode(s, *, padded=True, wrapcol=0): + return binascii.b2a_base32(s, padded=padded, wrapcol=wrapcol) +b32encode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32') - if not isinstance(s, bytes_types): - s = memoryview(s).tobytes() - leftover = len(s) % 5 - # Pad the last quantum with zero bits if necessary - if leftover: - s = s + b'\0' * (5 - leftover) # Don't use += ! - encoded = bytearray() - from_bytes = int.from_bytes - b32tab2 = _b32tab2[alphabet] - for i in range(0, len(s), 5): - c = from_bytes(s[i: i + 5]) # big endian - encoded += (b32tab2[c >> 30] + # bits 1 - 10 - b32tab2[(c >> 20) & 0x3ff] + # bits 11 - 20 - b32tab2[(c >> 10) & 0x3ff] + # bits 21 - 30 - b32tab2[c & 0x3ff] # bits 31 - 40 - ) - # Adjust for any leftover partial quanta - if leftover == 1: - encoded[-6:] = b'======' - elif leftover == 2: - encoded[-4:] = b'====' - elif leftover == 3: - encoded[-3:] = b'===' - elif leftover == 4: - encoded[-1:] = b'=' - return bytes(encoded) - -def _b32decode(alphabet, s, casefold=False, map01=None): - # Delay the initialization of the table to not waste memory - # if the function is never called - if alphabet not in _b32rev: - _b32rev[alphabet] = {v: k for k, v in enumerate(alphabet)} +def b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b'', + canonical=False): s = _bytes_from_decode_data(s) - if len(s) % 8: - raise binascii.Error('Incorrect padding') # Handle section 2.4 zero and one mapping. The flag map01 will be either # False, or the character to map the digit 1 (one) to. It should be # either L (el) or I (eye). if map01 is not None: map01 = _bytes_from_decode_data(map01) - assert len(map01) == 1, repr(map01) s = s.translate(bytes.maketrans(b'01', b'O' + map01)) if casefold: s = s.upper() - # Strip off pad characters from the right. We need to count the pad - # characters because this will tell us how many null bytes to remove from - # the end of the decoded string. - l = len(s) - s = s.rstrip(b'=') - padchars = l - len(s) - # Now decode the full quanta - decoded = bytearray() - b32rev = _b32rev[alphabet] - for i in range(0, len(s), 8): - quanta = s[i: i + 8] - acc = 0 - try: - for c in quanta: - acc = (acc << 5) + b32rev[c] - except KeyError: - raise binascii.Error('Non-base32 digit found') from None - decoded += acc.to_bytes(5) # big endian - # Process the last, partial quanta - if l % 8 or padchars not in {0, 1, 3, 4, 6}: - raise binascii.Error('Incorrect padding') - if padchars and decoded: - acc <<= 5 * padchars - last = acc.to_bytes(5) # big endian - leftover = (43 - 5 * padchars) // 8 # 1: 4, 3: 3, 4: 2, 6: 1 - decoded[-5:] = last[:leftover] - return bytes(decoded) - - -def b32encode(s): - return _b32encode(_b32alphabet, s) -b32encode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32') - -def b32decode(s, casefold=False, map01=None): - return _b32decode(_b32alphabet, s, casefold, map01) + return binascii.a2b_base32(s, padded=padded, ignorechars=ignorechars, + canonical=canonical) b32decode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32', extra_args=_B32_DECODE_MAP01_DOCSTRING) -def b32hexencode(s): - return _b32encode(_b32hexalphabet, s) +def b32hexencode(s, *, padded=True, wrapcol=0): + return binascii.b2a_base32(s, padded=padded, wrapcol=wrapcol, + alphabet=binascii.BASE32HEX_ALPHABET) b32hexencode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32hex') -def b32hexdecode(s, casefold=False): +def b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b'', + canonical=False): + s = _bytes_from_decode_data(s) # base32hex does not have the 01 mapping - return _b32decode(_b32hexalphabet, s, casefold) + if casefold: + s = s.upper() + return binascii.a2b_base32(s, alphabet=binascii.BASE32HEX_ALPHABET, + padded=padded, ignorechars=ignorechars, + canonical=canonical) b32hexdecode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32hex', extra_args='') @@ -264,62 +269,47 @@ def b32hexdecode(s, casefold=False): # RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns # lowercase. The RFC also recommends against accepting input case # insensitively. -def b16encode(s): +def b16encode(s, *, wrapcol=0): """Encode the bytes-like object s using Base16 and return a bytes object. + + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. """ - return binascii.hexlify(s).upper() + if not wrapcol: + return binascii.hexlify(s).upper() + if wrapcol < 0: + raise ValueError('Negative wrapcol') + if wrapcol < 2: + wrapcol = 2 + return binascii.hexlify(s, bytes_per_sep=-(wrapcol//2), sep=b'\n').upper() -def b16decode(s, casefold=False): +def b16decode(s, casefold=False, *, ignorechars=b''): """Decode the Base16 encoded bytes-like object or ASCII string s. Optional casefold is a flag specifying whether a lowercase alphabet is acceptable as input. For security purposes, the default is False. + ignorechars should be a byte string containing characters to ignore + from the input. + The result is returned as a bytes object. A binascii.Error is raised if s is incorrectly padded or if there are non-alphabet characters present in the input. """ - s = _bytes_from_decode_data(s) - if casefold: - s = s.upper() - if s.translate(None, delete=b'0123456789ABCDEF'): - raise binascii.Error('Non-base16 digit found') - return binascii.unhexlify(s) + if not casefold: + s = _bytes_from_decode_data(s) + if not isinstance(ignorechars, bytes): + ignorechars = bytes(memoryview(ignorechars)) + for b in b'abcdef': + if b in s and b not in ignorechars: + raise binascii.Error('Non-base16 digit found') + s = s.translate(None, delete=b'abcdef') + return binascii.unhexlify(s, ignorechars=ignorechars) # # Ascii85 encoding/decoding # - -_a85chars = None -_a85chars2 = None -_A85START = b"<~" -_A85END = b"~>" - -def _85encode(b, chars, chars2, pad=False, foldnuls=False, foldspaces=False): - # Helper function for a85encode and b85encode - if not isinstance(b, bytes_types): - b = memoryview(b).tobytes() - - padding = (-len(b)) % 4 - if padding: - b = b + b'\0' * padding - words = struct.Struct('!%dI' % (len(b) // 4)).unpack(b) - - chunks = [b'z' if foldnuls and not word else - b'y' if foldspaces and word == 0x20202020 else - (chars2[word // 614125] + - chars2[word // 85 % 7225] + - chars[word % 85]) - for word in words] - - if padding and not pad: - if chunks[-1] == b'z': - chunks[-1] = chars[0] * 5 - chunks[-1] = chunks[-1][:-padding] - - return b''.join(chunks) - def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): """Encode bytes-like object b using Ascii85 and return a bytes object. @@ -327,9 +317,8 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This feature is not supported by the "standard" Adobe encoding. - wrapcol controls whether the output should have newline (b'\\n') characters - added to it. If this is non-zero, each output line will be at most this - many characters long, excluding the trailing newline. + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. pad controls whether the input is padded to a multiple of 4 before encoding. Note that the btoa implementation always pads. @@ -337,31 +326,11 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): adobe controls whether the encoded byte sequence is framed with <~ and ~>, which is used by the Adobe implementation. """ - global _a85chars, _a85chars2 - # Delay the initialization of tables to not waste memory - # if the function is never called - if _a85chars2 is None: - _a85chars = [bytes((i,)) for i in range(33, 118)] - _a85chars2 = [(a + b) for a in _a85chars for b in _a85chars] + return binascii.b2a_ascii85(b, foldspaces=foldspaces, + adobe=adobe, wrapcol=wrapcol, pad=pad) - result = _85encode(b, _a85chars, _a85chars2, pad, True, foldspaces) - - if adobe: - result = _A85START + result - if wrapcol: - wrapcol = max(2 if adobe else 1, wrapcol) - chunks = [result[i: i + wrapcol] - for i in range(0, len(result), wrapcol)] - if adobe: - if len(chunks[-1]) + 2 > wrapcol: - chunks.append(b'') - result = b'\n'.join(chunks) - if adobe: - result += _A85END - - return result - -def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'): +def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v', + canonical=False): """Decode the Ascii85 encoded bytes-like object or ASCII string b. foldspaces is a flag that specifies whether the 'y' short sequence should be @@ -375,151 +344,56 @@ def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'): input. This should only contain whitespace characters, and by default contains all whitespace characters in ASCII. + If canonical is true, non-canonical encodings are rejected. + The result is returned as a bytes object. """ - b = _bytes_from_decode_data(b) - if adobe: - if not b.endswith(_A85END): - raise ValueError( - "Ascii85 encoded byte sequences must end " - "with {!r}".format(_A85END) - ) - if b.startswith(_A85START): - b = b[2:-2] # Strip off start/end markers - else: - b = b[:-2] - # - # We have to go through this stepwise, so as to ignore spaces and handle - # special short sequences - # - packI = struct.Struct('!I').pack - decoded = [] - decoded_append = decoded.append - curr = [] - curr_append = curr.append - curr_clear = curr.clear - for x in b + b'u' * 4: - if b'!'[0] <= x <= b'u'[0]: - curr_append(x) - if len(curr) == 5: - acc = 0 - for x in curr: - acc = 85 * acc + (x - 33) - try: - decoded_append(packI(acc)) - except struct.error: - raise ValueError('Ascii85 overflow') from None - curr_clear() - elif x == b'z'[0]: - if curr: - raise ValueError('z inside Ascii85 5-tuple') - decoded_append(b'\0\0\0\0') - elif foldspaces and x == b'y'[0]: - if curr: - raise ValueError('y inside Ascii85 5-tuple') - decoded_append(b'\x20\x20\x20\x20') - elif x in ignorechars: - # Skip whitespace - continue - else: - raise ValueError('Non-Ascii85 digit found: %c' % x) + return binascii.a2b_ascii85(b, foldspaces=foldspaces, + adobe=adobe, ignorechars=ignorechars, + canonical=canonical) - result = b''.join(decoded) - padding = 4 - len(curr) - if padding: - # Throw away the extra padding - result = result[:-padding] - return result - -# The following code is originally taken (with permission) from Mercurial - -_b85alphabet = (b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" - b"abcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~") -_b85chars = None -_b85chars2 = None -_b85dec = None - -def b85encode(b, pad=False): +def b85encode(b, pad=False, *, wrapcol=0): """Encode bytes-like object b in base85 format and return a bytes object. + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. + If pad is true, the input is padded with b'\\0' so its length is a multiple of 4 bytes before encoding. """ - global _b85chars, _b85chars2 - # Delay the initialization of tables to not waste memory - # if the function is never called - if _b85chars2 is None: - _b85chars = [bytes((i,)) for i in _b85alphabet] - _b85chars2 = [(a + b) for a in _b85chars for b in _b85chars] - return _85encode(b, _b85chars, _b85chars2, pad) + return binascii.b2a_base85(b, wrapcol=wrapcol, pad=pad) -def b85decode(b): +def b85decode(b, *, ignorechars=b'', canonical=False): """Decode the base85-encoded bytes-like object or ASCII string b + If canonical is true, non-canonical encodings are rejected. + The result is returned as a bytes object. """ - global _b85dec - # Delay the initialization of tables to not waste memory - # if the function is never called - if _b85dec is None: - _b85dec = [None] * 256 - for i, c in enumerate(_b85alphabet): - _b85dec[c] = i + return binascii.a2b_base85(b, ignorechars=ignorechars, + canonical=canonical) - b = _bytes_from_decode_data(b) - padding = (-len(b)) % 5 - b = b + b'~' * padding - out = [] - packI = struct.Struct('!I').pack - for i in range(0, len(b), 5): - chunk = b[i:i + 5] - acc = 0 - try: - for c in chunk: - acc = acc * 85 + _b85dec[c] - except TypeError: - for j, c in enumerate(chunk): - if _b85dec[c] is None: - raise ValueError('bad base85 character at position %d' - % (i + j)) from None - raise - try: - out.append(packI(acc)) - except struct.error: - raise ValueError('base85 overflow in hunk starting at byte %d' - % i) from None +def z85encode(s, pad=False, *, wrapcol=0): + """Encode bytes-like object b in z85 format and return a bytes object. - result = b''.join(out) - if padding: - result = result[:-padding] - return result + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. -_z85alphabet = (b'0123456789abcdefghijklmnopqrstuvwxyz' - b'ABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#') -# Translating b85 valid but z85 invalid chars to b'\x00' is required -# to prevent them from being decoded as b85 valid chars. -_z85_b85_decode_diff = b';_`|~' -_z85_decode_translation = bytes.maketrans( - _z85alphabet + _z85_b85_decode_diff, - _b85alphabet + b'\x00' * len(_z85_b85_decode_diff) -) -_z85_encode_translation = bytes.maketrans(_b85alphabet, _z85alphabet) + If pad is true, the input is padded with b'\\0' so its length is a multiple of + 4 bytes before encoding. + """ + return binascii.b2a_base85(s, wrapcol=wrapcol, pad=pad, + alphabet=binascii.Z85_ALPHABET) -def z85encode(s): - """Encode bytes-like object b in z85 format and return a bytes object.""" - return b85encode(s).translate(_z85_encode_translation) - -def z85decode(s): +def z85decode(s, *, ignorechars=b'', canonical=False): """Decode the z85-encoded bytes-like object or ASCII string b + If canonical is true, non-canonical encodings are rejected. + The result is returned as a bytes object. """ - s = _bytes_from_decode_data(s) - s = s.translate(_z85_decode_translation) - try: - return b85decode(s) - except ValueError as e: - raise ValueError(e.args[0].replace('base85', 'z85')) from None + return binascii.a2b_base85(s, alphabet=binascii.Z85_ALPHABET, + ignorechars=ignorechars, canonical=canonical) # Legacy interface. This code could be cleaned up since I don't believe # binascii has any line length limitations. It just doesn't seem worth it @@ -563,11 +437,10 @@ def encodebytes(s): """Encode a bytestring into a bytes object containing multiple lines of base-64 data.""" _input_type_check(s) - pieces = [] - for i in range(0, len(s), MAXBINSIZE): - chunk = s[i : i + MAXBINSIZE] - pieces.append(binascii.b2a_base64(chunk)) - return b"".join(pieces) + result = binascii.b2a_base64(s, wrapcol=MAXLINESIZE) + if result == b'\n': + return b'' + return result def decodebytes(s): @@ -601,7 +474,14 @@ def main(): with open(args[0], 'rb') as f: func(f, sys.stdout.buffer) else: - func(sys.stdin.buffer, sys.stdout.buffer) + if sys.stdin.isatty(): + # gh-138775: read terminal input data all at once to detect EOF + import io + data = sys.stdin.buffer.read() + buffer = io.BytesIO(data) + else: + buffer = sys.stdin.buffer + func(buffer, sys.stdout.buffer) if __name__ == '__main__': diff --git a/Lib/bdb.py b/Lib/bdb.py index efc3e0a235a..50cf2b3f5b3 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -199,6 +199,8 @@ def __init__(self, skip=None, backend='settrace'): self.frame_returning = None self.trace_opcodes = False self.enterframe = None + self.cmdframe = None + self.cmdlineno = None self.code_linenos = weakref.WeakKeyDictionary() self.backend = backend if backend == 'monitoring': @@ -297,7 +299,12 @@ def dispatch_line(self, frame): self.user_line(). Raise BdbQuit if self.quitting is set. Return self.trace_dispatch to continue tracing in this scope. """ - if self.stop_here(frame) or self.break_here(frame): + # GH-136057 + # For line events, we don't want to stop at the same line where + # the latest next/step command was issued. + if (self.stop_here(frame) or self.break_here(frame)) and not ( + self.cmdframe == frame and self.cmdlineno == frame.f_lineno + ): self.user_line(frame) self.restart_events() if self.quitting: raise BdbQuit @@ -526,7 +533,8 @@ def _set_trace_opcodes(self, trace_opcodes): if self.monitoring_tracer: self.monitoring_tracer.update_local_events() - def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): + def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False, + cmdframe=None, cmdlineno=None): """Set the attributes for stopping. If stoplineno is greater than or equal to 0, then stop at line @@ -539,6 +547,10 @@ def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): # stoplineno >= 0 means: stop at line >= the stoplineno # stoplineno -1 means: don't stop at all self.stoplineno = stoplineno + # cmdframe/cmdlineno is the frame/line number when the user issued + # step/next commands. + self.cmdframe = cmdframe + self.cmdlineno = cmdlineno self._set_trace_opcodes(opcode) def _set_caller_tracefunc(self, current_frame): @@ -564,7 +576,9 @@ def set_until(self, frame, lineno=None): def set_step(self): """Stop after one line of code.""" - self._set_stopinfo(None, None) + # set_step() could be called from signal handler so enterframe might be None + self._set_stopinfo(None, None, cmdframe=self.enterframe, + cmdlineno=getattr(self.enterframe, 'f_lineno', None)) def set_stepinstr(self): """Stop before the next instruction.""" @@ -572,7 +586,7 @@ def set_stepinstr(self): def set_next(self, frame): """Stop on the next line in or below the given frame.""" - self._set_stopinfo(frame, None) + self._set_stopinfo(frame, None, cmdframe=frame, cmdlineno=frame.f_lineno) def set_return(self, frame): """Stop when returning from the given frame.""" diff --git a/Lib/cProfile.py b/Lib/cProfile.py index 4af82f2cb8c..cc6255f61ae 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -9,6 +9,5 @@ __all__ = ["run", "runctx", "Profile"] if __name__ == "__main__": - import sys from profiling.tracing.__main__ import main main() diff --git a/Lib/calendar.py b/Lib/calendar.py index 678c7be5aac..d80c3fd9524 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -498,30 +498,29 @@ def formatday(self, day, weekday): """ if day == 0: # day outside month - return ' ' % self.cssclass_noday + return f' ' else: - return '%d' % (self.cssclasses[weekday], day) + return f'{day}' def formatweek(self, theweek): """ Return a complete week as a table row. """ s = ''.join(self.formatday(d, wd) for (d, wd) in theweek) - return '%s' % s + return f'{s}' def formatweekday(self, day): """ Return a weekday name as a table header. """ - return '%s' % ( - self.cssclasses_weekday_head[day], day_abbr[day]) + return f'{day_abbr[day]}' def formatweekheader(self): """ Return a header for a week as a table row. """ s = ''.join(self.formatweekday(i) for i in self.iterweekdays()) - return '%s' % s + return f'{s}' def formatmonthname(self, theyear, themonth, withyear=True): """ @@ -529,11 +528,10 @@ def formatmonthname(self, theyear, themonth, withyear=True): """ _validate_month(themonth) if withyear: - s = '%s %s' % (standalone_month_name[themonth], theyear) + s = f'{standalone_month_name[themonth]} {theyear}' else: s = standalone_month_name[themonth] - return '%s' % ( - self.cssclass_month_head, s) + return f'{s}' def formatmonth(self, theyear, themonth, withyear=True): """ @@ -541,8 +539,7 @@ def formatmonth(self, theyear, themonth, withyear=True): """ v = [] a = v.append - a('' % ( - self.cssclass_month)) + a(f'
') a('\n') a(self.formatmonthname(theyear, themonth, withyear=withyear)) a('\n') @@ -562,11 +559,9 @@ def formatyear(self, theyear, width=3): v = [] a = v.append width = max(width, 1) - a('
' % - self.cssclass_year) + a(f'
') a('\n') - a('' % ( - width, self.cssclass_year_head, theyear)) + a(f'') for i in range(JANUARY, JANUARY+12, width): # months in this row months = range(i, min(i+width, 13)) @@ -579,29 +574,48 @@ def formatyear(self, theyear, width=3): a('
%s
{theyear}
') return ''.join(v) - def formatyearpage(self, theyear, width=3, css='calendar.css', encoding=None): + def _format_html_page(self, theyear, content, css, encoding): """ - Return a formatted year as a complete HTML page. + Return a complete HTML page with the given content. """ if encoding is None: encoding = 'utf-8' v = [] a = v.append - a('\n' % encoding) - a('\n') - a('\n') + a('\n') + a('\n') a('\n') - a('\n' % encoding) + a(f'\n') + a('\n') + a(f'Calendar for {theyear}\n') + a('\n') if css is not None: - a('\n' % css) - a('Calendar for %d\n' % theyear) + a(f'\n') a('\n') a('\n') - a(self.formatyear(theyear, width)) + a(content) a('\n') a('\n') return ''.join(v).encode(encoding, "xmlcharrefreplace") + def formatyearpage(self, theyear, width=3, css='calendar.css', encoding=None): + """ + Return a formatted year as a complete HTML page. + """ + content = self.formatyear(theyear, width) + return self._format_html_page(theyear, content, css, encoding) + + def formatmonthpage(self, theyear, themonth, width=3, css='calendar.css', encoding=None): + """ + Return a formatted month as a complete HTML page. + """ + content = self.formatmonth(theyear, themonth, width) + return self._format_html_page(theyear, content, css, encoding) + class different_locale: def __init__(self, locale): @@ -886,7 +900,7 @@ def main(args=None): parser.add_argument( "month", nargs='?', type=int, - help="month number (1-12, text only)" + help="month number (1-12)" ) options = parser.parse_args(args) @@ -899,9 +913,6 @@ def main(args=None): today = datetime.date.today() if options.type == "html": - if options.month: - parser.error("incorrect number of arguments") - sys.exit(1) if options.locale: cal = LocaleHTMLCalendar(locale=locale) else: @@ -912,10 +923,14 @@ def main(args=None): encoding = 'utf-8' optdict = dict(encoding=encoding, css=options.css) write = sys.stdout.buffer.write + if options.year is None: write(cal.formatyearpage(today.year, **optdict)) else: - write(cal.formatyearpage(options.year, **optdict)) + if options.month: + write(cal.formatmonthpage(options.year, options.month, **optdict)) + else: + write(cal.formatyearpage(options.year, **optdict)) else: if options.locale: cal = _CLIDemoLocaleCalendar(highlight_day=today, locale=locale) diff --git a/Lib/codeop.py b/Lib/codeop.py index 8cac00442d9..40e88423119 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -66,9 +66,9 @@ def _maybe_compile(compiler, source, filename, symbol, flags): try: compiler(source + "\n", filename, symbol, flags=flags) return None - except _IncompleteInputError as e: + except _IncompleteInputError: return None - except SyntaxError as e: + except SyntaxError: pass # fallthrough diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index b8653f40a94..20f1e728733 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -32,6 +32,8 @@ _sys.modules['collections.abc'] = _collections_abc abc = _collections_abc +lazy from copy import copy as _copy +lazy from heapq import nlargest as _nlargest from itertools import chain as _chain from itertools import repeat as _repeat from itertools import starmap as _starmap @@ -59,8 +61,6 @@ except ImportError: pass -heapq = None # Lazily imported - ################################################################################ ### OrderedDict @@ -328,14 +328,14 @@ def __ior__(self, other): return self def __or__(self, other): - if not isinstance(other, dict): + if not isinstance(other, (dict, frozendict)): return NotImplemented new = self.__class__(self) new.update(other) return new def __ror__(self, other): - if not isinstance(other, dict): + if not isinstance(other, (dict, frozendict)): return NotImplemented new = self.__class__(other) new.update(self) @@ -634,12 +634,7 @@ def most_common(self, n=None): if n is None: return sorted(self.items(), key=_itemgetter(1), reverse=True) - # Lazy import to speedup Python startup time - global heapq - if heapq is None: - import heapq - - return heapq.nlargest(n, self.items(), key=_itemgetter(1)) + return _nlargest(n, self.items(), key=_itemgetter(1)) def elements(self): '''Iterator over elements repeating each as many times as its count. @@ -796,6 +791,7 @@ def __repr__(self): # set(cp - cq) == sp - sq # set(cp | cq) == sp | sq # set(cp & cq) == sp & sq + # set(cp ^ cq) == sp ^ sq def __eq__(self, other): 'True if all counts agree. Missing counts are treated as zero.' @@ -908,6 +904,33 @@ def __and__(self, other): result[elem] = newcount return result + def __xor__(self, other): + '''Symmetric difference. Absolute value of count differences. + + The symmetric difference p ^ q is equivalent to: + + (p - q) | (q - p). + + For each element, symmetric difference gives the same result as: + + max(p[elem], q[elem]) - min(p[elem], q[elem]) + + >>> Counter(a=5, b=3, c=2, d=2) ^ Counter(a=1, b=3, c=5, e=1) + Counter({'a': 4, 'c': 3, 'd': 2, 'e': 1}) + + ''' + if not isinstance(other, Counter): + return NotImplemented + result = Counter() + for elem, count in self.items(): + newcount = abs(count - other[elem]) + if newcount: + result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count: + result[elem] = abs(count) + return result + def __pos__(self): 'Adds an empty counter, effectively stripping negative and zero counts' result = Counter() @@ -990,6 +1013,22 @@ def __iand__(self, other): self[elem] = other_count return self._keep_positive() + def __ixor__(self, other): + '''Inplace symmetric difference. Absolute value of count differences. + + >>> c = Counter(a=5, b=3, c=2, d=2) + >>> c ^= Counter(a=1, b=3, c=5, e=1) + >>> c + Counter({'a': 4, 'c': 3, 'd': 2, 'e': 1}) + + ''' + for elem, count in self.items(): + self[elem] = abs(count - other[elem]) + for elem, count in other.items(): + if elem not in self: + self[elem] = abs(count) + return self._keep_positive() + ######################################################################## ### ChainMap @@ -1177,14 +1216,14 @@ def __repr__(self): def __or__(self, other): if isinstance(other, UserDict): return self.__class__(self.data | other.data) - if isinstance(other, dict): + if isinstance(other, (dict, frozendict)): return self.__class__(self.data | other) return NotImplemented def __ror__(self, other): if isinstance(other, UserDict): return self.__class__(other.data | self.data) - if isinstance(other, dict): + if isinstance(other, (dict, frozendict)): return self.__class__(other | self.data) return NotImplemented @@ -1205,16 +1244,29 @@ def __copy__(self): def copy(self): if self.__class__ is UserDict: return UserDict(self.data.copy()) - import copy data = self.data try: self.data = {} - c = copy.copy(self) + c = _copy(self) finally: self.data = data c.update(self) return c + + # This method has a default implementation in MutableMapping, but dict's + # equivalent is last-in, first-out instead of first-in, first-out. + def popitem(self): + """Remove and return a (key, value) pair as a 2-tuple. + + Removes pairs in the same order as the wrapped mapping's popitem() + method. For dict objects (the default), that order is last-in, + first-out (LIFO). + Raises KeyError if the UserDict is empty. + """ + return self.data.popitem() + + @classmethod def fromkeys(cls, iterable, value=None): d = cls() @@ -1498,6 +1550,8 @@ def format_map(self, mapping): return self.data.format_map(mapping) def index(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data return self.data.index(sub, start, end) def isalpha(self): @@ -1566,6 +1620,8 @@ def rfind(self, sub, start=0, end=_sys.maxsize): return self.data.rfind(sub, start, end) def rindex(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data return self.data.rindex(sub, start, end) def rjust(self, width, *args): diff --git a/Lib/compileall.py b/Lib/compileall.py index 67fe370451e..c452aed1358 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -165,6 +165,14 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, stripdir = os.fspath(stripdir) if stripdir is not None else None name = os.path.basename(fullname) + # Without a cache_tag, we can only create legacy .pyc files. None of our + # callers seem to expect this, so the best we can do is fail without raising + if not legacy and sys.implementation.cache_tag is None: + if not quiet: + print("No cache tag is available to generate .pyc path for", + repr(fullname)) + return False + dfile = None if ddir is not None: @@ -223,7 +231,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, cfile = importlib.util.cache_from_source(fullname) opt_cfiles[opt_level] = cfile - head, tail = name[:-3], name[-3:] + tail = name[-3:] if tail == '.py': if not force: try: diff --git a/Lib/concurrent/futures/interpreter.py b/Lib/concurrent/futures/interpreter.py index 53c6e757ded..85c1da2c722 100644 --- a/Lib/concurrent/futures/interpreter.py +++ b/Lib/concurrent/futures/interpreter.py @@ -2,7 +2,6 @@ from concurrent import interpreters import sys -import textwrap from . import thread as _thread import traceback diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index a14650bf5fa..a42afa68efc 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -474,9 +474,23 @@ def _terminate_broken(self, cause): bpe = BrokenProcessPool("A process in the process pool was " "terminated abruptly while the future was " "running or pending.") + cause_str = None if cause is not None: - bpe.__cause__ = _RemoteTraceback( - f"\n'''\n{''.join(cause)}'''") + cause_str = ''.join(cause) + else: + # No cause known, so report any processes that have + # terminated with nonzero exit codes, e.g. from a + # segfault. Multiple may terminate simultaneously, + # so include all of them in the traceback. + errors = [] + for p in self.processes.values(): + if p.exitcode is not None and p.exitcode != 0: + errors.append(f"Process {p.pid} terminated abruptly " + f"with exit code {p.exitcode}") + if errors: + cause_str = "\n".join(errors) + if cause_str: + bpe.__cause__ = _RemoteTraceback(f"\n'''\n{cause_str}'''") # Mark pending tasks as failed. for work_id, work_item in self.pending_work_items.items(): diff --git a/Lib/concurrent/interpreters/__init__.py b/Lib/concurrent/interpreters/__init__.py index aa46a2b37a4..ea4147ee9a2 100644 --- a/Lib/concurrent/interpreters/__init__.py +++ b/Lib/concurrent/interpreters/__init__.py @@ -149,12 +149,17 @@ def __del__(self): def __reduce__(self): return (type(self), (self._id,)) - def _decref(self): + # gh-135729: Globals might be destroyed by the time this is called, so we + # need to keep references ourself + def _decref(self, *, + InterpreterNotFoundError=InterpreterNotFoundError, + _interp_decref=_interpreters.decref, + ): if not self._ownsref: return self._ownsref = False try: - _interpreters.decref(self._id) + _interp_decref(self._id) except InterpreterNotFoundError: pass diff --git a/Lib/concurrent/interpreters/_queues.py b/Lib/concurrent/interpreters/_queues.py index b5cc0b89449..ee159d7de63 100644 --- a/Lib/concurrent/interpreters/_queues.py +++ b/Lib/concurrent/interpreters/_queues.py @@ -223,7 +223,7 @@ def put(self, obj, block=True, timeout=None, *, while True: try: _queues.put(self._id, obj, unboundop) - except QueueFull as exc: + except QueueFull: if timeout is not None and time.time() >= end: raise # re-raise time.sleep(_delay) @@ -258,7 +258,7 @@ def get(self, block=True, timeout=None, *, while True: try: obj, unboundop = _queues.get(self._id) - except QueueEmpty as exc: + except QueueEmpty: if timeout is not None and time.time() >= end: raise # re-raise time.sleep(_delay) @@ -277,7 +277,7 @@ def get_nowait(self): """ try: obj, unboundop = _queues.get(self._id) - except QueueEmpty as exc: + except QueueEmpty: raise # re-raise if unboundop is not None: assert obj is None, repr(obj) diff --git a/Lib/configparser.py b/Lib/configparser.py index 18af1eadaad..a53ac872764 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -315,12 +315,15 @@ def __init__(self, source, *args): def append(self, lineno, line): self.errors.append((lineno, line)) - self.message += '\n\t[line %2d]: %s' % (lineno, repr(line)) + self.message += f'\n\t[line {lineno:2d}]: {line!r}' def combine(self, others): + messages = [self.message] for other in others: - for error in other.errors: - self.append(*error) + for lineno, line in other.errors: + self.errors.append((lineno, line)) + messages.append(f'\n\t[line {lineno:2d}]: {line!r}') + self.message = "".join(messages) return self @staticmethod @@ -613,7 +616,9 @@ class RawConfigParser(MutableMapping): \] # ] """ _OPT_TMPL = r""" - (?P